diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 05aa600e649e..bb6bc325a3bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,7 @@ on: - try - try-perf - automation/bors/try + - automation/bors/auto pull_request: branches: - "**" @@ -56,7 +57,7 @@ jobs: - name: Test citool # Only test citool on the auto branch, to reduce latency of the calculate matrix job # on PR/try builds. - if: ${{ github.ref == 'refs/heads/auto' }} + if: ${{ github.ref == 'refs/heads/auto' || github.ref == 'refs/heads/automation/bors/auto' }} run: | cd src/ci/citool CARGO_INCREMENTAL=0 cargo test @@ -79,7 +80,7 @@ jobs: # access the environment. # # We only enable the environment for the rust-lang/rust repository, so that CI works on forks. - environment: ${{ ((github.repository == 'rust-lang/rust' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf' || github.ref == 'refs/heads/automation/bors/try' || github.ref == 'refs/heads/auto')) && 'bors') || '' }} + environment: ${{ ((github.repository == 'rust-lang/rust' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf' || github.ref == 'refs/heads/automation/bors/try' || github.ref == 'refs/heads/auto' || github.ref == 'refs/heads/automation/bors/auto')) && 'bors') || '' }} env: CI_JOB_NAME: ${{ matrix.name }} CI_JOB_DOC_URL: ${{ matrix.doc_url }} @@ -313,7 +314,7 @@ jobs: needs: [ calculate_matrix, job ] # !cancelled() executes the job regardless of whether the previous jobs passed or failed if: ${{ !cancelled() && contains(fromJSON('["auto", "try"]'), needs.calculate_matrix.outputs.run_type) }} - environment: ${{ ((github.repository == 'rust-lang/rust' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf' || github.ref == 'refs/heads/automation/bors/try' || github.ref == 'refs/heads/auto')) && 'bors') || '' }} + environment: ${{ ((github.repository == 'rust-lang/rust' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf' || github.ref == 'refs/heads/automation/bors/try' || github.ref == 'refs/heads/auto' || github.ref == 'refs/heads/automation/bors/auto')) && 'bors') || '' }} steps: - name: checkout the source code uses: actions/checkout@v5 diff --git a/Cargo.lock b/Cargo.lock index 1cd998a0bd2f..587824617070 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -666,6 +666,7 @@ dependencies = [ "indoc", "itertools", "opener", + "rustc-literal-escaper", "walkdir", ] @@ -3637,8 +3638,6 @@ dependencies = [ "rustc_span", "rustc_symbol_mangling", "rustc_target", - "serde", - "serde_json", "smallvec", "tracing", ] @@ -4145,8 +4144,8 @@ version = "0.0.0" dependencies = [ "expect-test", "memchr", + "unicode-ident", "unicode-properties", - "unicode-xid", ] [[package]] @@ -5357,9 +5356,9 @@ dependencies = [ [[package]] name = "stringdex" -version = "0.0.3" +version = "0.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556a6126952cb2f5150057c98a77cc6c771027dea2825bf7fa03d3d638b0a4f8" +checksum = "07ab85c3f308f022ce6861ab57576b5b6ebc4835f9577e67e0f35f6c351e3f0a" dependencies = [ "stacker", ] @@ -5982,24 +5981,24 @@ checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unicode-normalization" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" dependencies = [ "tinyvec", ] [[package]] name = "unicode-properties" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" +checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "unicode-script" diff --git a/compiler/rustc_abi/src/layout.rs b/compiler/rustc_abi/src/layout.rs index b852d71b8008..4f1594d02a82 100644 --- a/compiler/rustc_abi/src/layout.rs +++ b/compiler/rustc_abi/src/layout.rs @@ -714,7 +714,7 @@ impl LayoutCalculator { }, fields: FieldsShape::Arbitrary { offsets: [niche_offset].into(), - memory_index: [0].into(), + in_memory_order: [FieldIdx::new(0)].into(), }, backend_repr: abi, largest_niche, @@ -1008,8 +1008,8 @@ impl LayoutCalculator { let pair = LayoutData::::scalar_pair(&self.cx, tag, prim_scalar); let pair_offsets = match pair.fields { - FieldsShape::Arbitrary { ref offsets, ref memory_index } => { - assert_eq!(memory_index.raw, [0, 1]); + FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => { + assert_eq!(in_memory_order.raw, [FieldIdx::new(0), FieldIdx::new(1)]); offsets } _ => panic!("encountered a non-arbitrary layout during enum layout"), @@ -1061,7 +1061,7 @@ impl LayoutCalculator { }, fields: FieldsShape::Arbitrary { offsets: [Size::ZERO].into(), - memory_index: [0].into(), + in_memory_order: [FieldIdx::new(0)].into(), }, largest_niche, uninhabited, @@ -1110,10 +1110,10 @@ impl LayoutCalculator { let pack = repr.pack; let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align }; let mut max_repr_align = repr.align; - let mut inverse_memory_index: IndexVec = fields.indices().collect(); + let mut in_memory_order: IndexVec = fields.indices().collect(); let optimize_field_order = !repr.inhibit_struct_field_reordering(); let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() }; - let optimizing = &mut inverse_memory_index.raw[..end]; + let optimizing = &mut in_memory_order.raw[..end]; let fields_excluding_tail = &fields.raw[..end]; // unsizable tail fields are excluded so that we use the same seed for the sized and unsized layouts. let field_seed = fields_excluding_tail @@ -1248,12 +1248,10 @@ impl LayoutCalculator { // regardless of the status of `-Z randomize-layout` } } - // inverse_memory_index holds field indices by increasing memory offset. - // That is, if field 5 has offset 0, the first element of inverse_memory_index is 5. + // in_memory_order holds field indices by increasing memory offset. + // That is, if field 5 has offset 0, the first element of in_memory_order is 5. // We now write field offsets to the corresponding offset slot; // field 5 with offset 0 puts 0 in offsets[5]. - // At the bottom of this function, we invert `inverse_memory_index` to - // produce `memory_index` (see `invert_mapping`). let mut unsized_field = None::<&F>; let mut offsets = IndexVec::from_elem(Size::ZERO, fields); let mut offset = Size::ZERO; @@ -1265,7 +1263,7 @@ impl LayoutCalculator { align = align.max(prefix_align); offset = prefix_size.align_to(prefix_align); } - for &i in &inverse_memory_index { + for &i in &in_memory_order { let field = &fields[i]; if let Some(unsized_field) = unsized_field { return Err(LayoutCalculatorError::UnexpectedUnsized(*unsized_field)); @@ -1322,18 +1320,6 @@ impl LayoutCalculator { debug!("univariant min_size: {:?}", offset); let min_size = offset; - // As stated above, inverse_memory_index holds field indices by increasing offset. - // This makes it an already-sorted view of the offsets vec. - // To invert it, consider: - // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0. - // Field 5 would be the first element, so memory_index is i: - // Note: if we didn't optimize, it's already right. - let memory_index = if optimize_field_order { - inverse_memory_index.invert_bijective_mapping() - } else { - debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices())); - inverse_memory_index.into_iter().map(|it| it.index() as u32).collect() - }; let size = min_size.align_to(align); // FIXME(oli-obk): deduplicate and harden these checks if size.bytes() >= dl.obj_size_bound() { @@ -1389,8 +1375,11 @@ impl LayoutCalculator { let pair = LayoutData::::scalar_pair(&self.cx, a, b); let pair_offsets = match pair.fields { - FieldsShape::Arbitrary { ref offsets, ref memory_index } => { - assert_eq!(memory_index.raw, [0, 1]); + FieldsShape::Arbitrary { ref offsets, ref in_memory_order } => { + assert_eq!( + in_memory_order.raw, + [FieldIdx::new(0), FieldIdx::new(1)] + ); offsets } FieldsShape::Primitive @@ -1434,7 +1423,7 @@ impl LayoutCalculator { Ok(LayoutData { variants: Variants::Single { index: VariantIdx::new(0) }, - fields: FieldsShape::Arbitrary { offsets, memory_index }, + fields: FieldsShape::Arbitrary { offsets, in_memory_order }, backend_repr: abi, largest_niche, uninhabited, @@ -1530,7 +1519,10 @@ where Ok(LayoutData { variants: Variants::Single { index: VariantIdx::new(0) }, - fields: FieldsShape::Arbitrary { offsets: [Size::ZERO].into(), memory_index: [0].into() }, + fields: FieldsShape::Arbitrary { + offsets: [Size::ZERO].into(), + in_memory_order: [FieldIdx::new(0)].into(), + }, backend_repr: repr, largest_niche: elt.largest_niche, uninhabited: false, diff --git a/compiler/rustc_abi/src/layout/coroutine.rs b/compiler/rustc_abi/src/layout/coroutine.rs index 2b22276d4aed..815cf1e28a08 100644 --- a/compiler/rustc_abi/src/layout/coroutine.rs +++ b/compiler/rustc_abi/src/layout/coroutine.rs @@ -182,33 +182,29 @@ pub(super) fn layout< // CoroutineLayout. debug!("prefix = {:#?}", prefix); let (outer_fields, promoted_offsets, promoted_memory_index) = match prefix.fields { - FieldsShape::Arbitrary { mut offsets, memory_index } => { - let mut inverse_memory_index = memory_index.invert_bijective_mapping(); - + FieldsShape::Arbitrary { mut offsets, in_memory_order } => { // "a" (`0..b_start`) and "b" (`b_start..`) correspond to // "outer" and "promoted" fields respectively. let b_start = tag_index.plus(1); let offsets_b = IndexVec::from_raw(offsets.raw.split_off(b_start.index())); let offsets_a = offsets; - // Disentangle the "a" and "b" components of `inverse_memory_index` + // Disentangle the "a" and "b" components of `in_memory_order` // by preserving the order but keeping only one disjoint "half" each. // FIXME(eddyb) build a better abstraction for permutations, if possible. - let inverse_memory_index_b: IndexVec = inverse_memory_index - .iter() - .filter_map(|&i| i.index().checked_sub(b_start.index()).map(FieldIdx::new)) - .collect(); - inverse_memory_index.raw.retain(|&i| i.index() < b_start.index()); - let inverse_memory_index_a = inverse_memory_index; - - // Since `inverse_memory_index_{a,b}` each only refer to their - // respective fields, they can be safely inverted - let memory_index_a = inverse_memory_index_a.invert_bijective_mapping(); - let memory_index_b = inverse_memory_index_b.invert_bijective_mapping(); + let mut in_memory_order_a = IndexVec::::new(); + let mut in_memory_order_b = IndexVec::::new(); + for i in in_memory_order { + if let Some(j) = i.index().checked_sub(b_start.index()) { + in_memory_order_b.push(FieldIdx::new(j)); + } else { + in_memory_order_a.push(i); + } + } let outer_fields = - FieldsShape::Arbitrary { offsets: offsets_a, memory_index: memory_index_a }; - (outer_fields, offsets_b, memory_index_b) + FieldsShape::Arbitrary { offsets: offsets_a, in_memory_order: in_memory_order_a }; + (outer_fields, offsets_b, in_memory_order_b.invert_bijective_mapping()) } _ => unreachable!(), }; @@ -236,7 +232,7 @@ pub(super) fn layout< )?; variant.variants = Variants::Single { index }; - let FieldsShape::Arbitrary { offsets, memory_index } = variant.fields else { + let FieldsShape::Arbitrary { offsets, in_memory_order } = variant.fields else { unreachable!(); }; @@ -249,8 +245,9 @@ pub(super) fn layout< // promoted fields were being used, but leave the elements not in the // subset as `invalid_field_idx`, which we can filter out later to // obtain a valid (bijective) mapping. + let memory_index = in_memory_order.invert_bijective_mapping(); let invalid_field_idx = promoted_memory_index.len() + memory_index.len(); - let mut combined_inverse_memory_index = + let mut combined_in_memory_order = IndexVec::from_elem_n(FieldIdx::new(invalid_field_idx), invalid_field_idx); let mut offsets_and_memory_index = iter::zip(offsets, memory_index); @@ -268,19 +265,18 @@ pub(super) fn layout< (promoted_offsets[field_idx], promoted_memory_index[field_idx]) } }; - combined_inverse_memory_index[memory_index] = i; + combined_in_memory_order[memory_index] = i; offset }) .collect(); - // Remove the unused slots and invert the mapping to obtain the - // combined `memory_index` (also see previous comment). - combined_inverse_memory_index.raw.retain(|&i| i.index() != invalid_field_idx); - let combined_memory_index = combined_inverse_memory_index.invert_bijective_mapping(); + // Remove the unused slots to obtain the combined `in_memory_order` + // (also see previous comment). + combined_in_memory_order.raw.retain(|&i| i.index() != invalid_field_idx); variant.fields = FieldsShape::Arbitrary { offsets: combined_offsets, - memory_index: combined_memory_index, + in_memory_order: combined_in_memory_order, }; size = size.max(variant.size); diff --git a/compiler/rustc_abi/src/layout/simple.rs b/compiler/rustc_abi/src/layout/simple.rs index b3807c872739..3784611b157b 100644 --- a/compiler/rustc_abi/src/layout/simple.rs +++ b/compiler/rustc_abi/src/layout/simple.rs @@ -16,7 +16,7 @@ impl LayoutData { variants: Variants::Single { index: VariantIdx::new(0) }, fields: FieldsShape::Arbitrary { offsets: IndexVec::new(), - memory_index: IndexVec::new(), + in_memory_order: IndexVec::new(), }, backend_repr: BackendRepr::Memory { sized }, largest_niche: None, @@ -108,7 +108,7 @@ impl LayoutData { variants: Variants::Single { index: VariantIdx::new(0) }, fields: FieldsShape::Arbitrary { offsets: [Size::ZERO, b_offset].into(), - memory_index: [0, 1].into(), + in_memory_order: [FieldIdx::new(0), FieldIdx::new(1)].into(), }, backend_repr: BackendRepr::ScalarPair(a, b), largest_niche, @@ -133,7 +133,7 @@ impl LayoutData { Some(fields) => FieldsShape::Union(fields), None => FieldsShape::Arbitrary { offsets: IndexVec::new(), - memory_index: IndexVec::new(), + in_memory_order: IndexVec::new(), }, }, backend_repr: BackendRepr::Memory { sized: true }, diff --git a/compiler/rustc_abi/src/lib.rs b/compiler/rustc_abi/src/lib.rs index 53b2c3f36dfc..061ad8617893 100644 --- a/compiler/rustc_abi/src/lib.rs +++ b/compiler/rustc_abi/src/lib.rs @@ -1636,19 +1636,14 @@ pub enum FieldsShape { // FIXME(eddyb) use small vector optimization for the common case. offsets: IndexVec, - /// Maps source order field indices to memory order indices, + /// Maps memory order field indices to source order indices, /// depending on how the fields were reordered (if at all). /// This is a permutation, with both the source order and the /// memory order using the same (0..n) index ranges. /// - /// Note that during computation of `memory_index`, sometimes - /// it is easier to operate on the inverse mapping (that is, - /// from memory order to source order), and that is usually - /// named `inverse_memory_index`. - /// // FIXME(eddyb) build a better abstraction for permutations, if possible. // FIXME(camlorn) also consider small vector optimization here. - memory_index: IndexVec, + in_memory_order: IndexVec, }, } @@ -1682,51 +1677,17 @@ impl FieldsShape { } } - #[inline] - pub fn memory_index(&self, i: usize) -> usize { - match *self { - FieldsShape::Primitive => { - unreachable!("FieldsShape::memory_index: `Primitive`s have no fields") - } - FieldsShape::Union(_) | FieldsShape::Array { .. } => i, - FieldsShape::Arbitrary { ref memory_index, .. } => { - memory_index[FieldIdx::new(i)].try_into().unwrap() - } - } - } - /// Gets source indices of the fields by increasing offsets. #[inline] pub fn index_by_increasing_offset(&self) -> impl ExactSizeIterator { - let mut inverse_small = [0u8; 64]; - let mut inverse_big = IndexVec::new(); - let use_small = self.count() <= inverse_small.len(); - - // We have to write this logic twice in order to keep the array small. - if let FieldsShape::Arbitrary { ref memory_index, .. } = *self { - if use_small { - for (field_idx, &mem_idx) in memory_index.iter_enumerated() { - inverse_small[mem_idx as usize] = field_idx.index() as u8; - } - } else { - inverse_big = memory_index.invert_bijective_mapping(); - } - } - // Primitives don't really have fields in the way that structs do, // but having this return an empty iterator for them is unhelpful // since that makes them look kinda like ZSTs, which they're not. let pseudofield_count = if let FieldsShape::Primitive = self { 1 } else { self.count() }; - (0..pseudofield_count).map(move |i| match *self { + (0..pseudofield_count).map(move |i| match self { FieldsShape::Primitive | FieldsShape::Union(_) | FieldsShape::Array { .. } => i, - FieldsShape::Arbitrary { .. } => { - if use_small { - inverse_small[i] as usize - } else { - inverse_big[i as u32].index() - } - } + FieldsShape::Arbitrary { in_memory_order, .. } => in_memory_order[i as u32].index(), }) } } diff --git a/compiler/rustc_arena/src/lib.rs b/compiler/rustc_arena/src/lib.rs index 5151c358774e..a0350dd6ff31 100644 --- a/compiler/rustc_arena/src/lib.rs +++ b/compiler/rustc_arena/src/lib.rs @@ -10,7 +10,6 @@ // tidy-alphabetical-start #![allow(clippy::mut_from_ref)] // Arena allocators are one place where this pattern is fine. #![allow(internal_features)] -#![cfg_attr(bootstrap, feature(maybe_uninit_slice))] #![cfg_attr(test, feature(test))] #![deny(unsafe_op_in_unsafe_fn)] #![doc(test(no_crate_inject, attr(deny(warnings), allow(internal_features))))] diff --git a/compiler/rustc_ast/src/ast.rs b/compiler/rustc_ast/src/ast.rs index 1ea9e0ae7718..7c922417ee29 100644 --- a/compiler/rustc_ast/src/ast.rs +++ b/compiler/rustc_ast/src/ast.rs @@ -1810,7 +1810,7 @@ pub enum ExprKind { /// or a `gen` block (`gen move { ... }`). /// /// The span is the "decl", which is the header before the body `{ }` - /// including the `asyng`/`gen` keywords and possibly `move`. + /// including the `async`/`gen` keywords and possibly `move`. Gen(CaptureBy, Box, GenBlockKind, Span), /// An await expression (`my_future.await`). Span is of await keyword. Await(Box, Span), diff --git a/compiler/rustc_ast_lowering/messages.ftl b/compiler/rustc_ast_lowering/messages.ftl index 370b15d2871a..0cd5c4f303de 100644 --- a/compiler/rustc_ast_lowering/messages.ftl +++ b/compiler/rustc_ast_lowering/messages.ftl @@ -6,6 +6,7 @@ ast_lowering_abi_specified_multiple_times = ast_lowering_arbitrary_expression_in_pattern = arbitrary expressions aren't allowed in patterns .pattern_from_macro_note = the `expr` fragment specifier forces the metavariable's content to be an expression + .const_block_in_pattern_help = use a named `const`-item or an `if`-guard (`x if x == const {"{ ... }"}`) instead ast_lowering_argument = argument @@ -56,6 +57,8 @@ ast_lowering_coroutine_too_many_parameters = ast_lowering_default_field_in_tuple = default fields are not supported in tuple structs .label = default fields are only supported on structs +ast_lowering_delegation_cycle_in_signature_resolution = encountered a cycle during delegation signature resolution +ast_lowering_delegation_unresolved_callee = failed to resolve delegation callee ast_lowering_does_not_support_modifiers = the `{$class_name}` register class does not support template modifiers diff --git a/compiler/rustc_ast_lowering/src/delegation.rs b/compiler/rustc_ast_lowering/src/delegation.rs index 82bade8829a2..5d2531e50393 100644 --- a/compiler/rustc_ast_lowering/src/delegation.rs +++ b/compiler/rustc_ast_lowering/src/delegation.rs @@ -44,17 +44,20 @@ use hir::{BodyId, HirId}; use rustc_abi::ExternAbi; use rustc_ast::*; use rustc_attr_parsing::{AttributeParser, ShouldEmit}; +use rustc_data_structures::fx::FxHashSet; use rustc_errors::ErrorGuaranteed; use rustc_hir::Target; use rustc_hir::attrs::{AttributeKind, InlineAttr}; -use rustc_hir::def_id::DefId; +use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_middle::span_bug; -use rustc_middle::ty::{Asyncness, DelegationFnSigAttrs, ResolverAstLowering}; +use rustc_middle::ty::{Asyncness, DelegationAttrs, DelegationFnSigAttrs, ResolverAstLowering}; use rustc_span::symbol::kw; use rustc_span::{DUMMY_SP, Ident, Span, Symbol}; +use smallvec::SmallVec; use {rustc_ast as ast, rustc_hir as hir}; use super::{GenericArgsMode, ImplTraitContext, LoweringContext, ParamMode}; +use crate::errors::{CycleInDelegationSignatureResolution, UnresolvedDelegationCallee}; use crate::{AllowReturnTypeNotation, ImplTraitPosition, ResolverAstLoweringExt}; pub(crate) struct DelegationResults<'hir> { @@ -64,24 +67,24 @@ pub(crate) struct DelegationResults<'hir> { pub generics: &'hir hir::Generics<'hir>, } -struct AttributeAdditionInfo { +struct AttrAdditionInfo { pub equals: fn(&hir::Attribute) -> bool, - pub kind: AttributeAdditionKind, + pub kind: AttrAdditionKind, } -enum AttributeAdditionKind { +enum AttrAdditionKind { Default { factory: fn(Span) -> hir::Attribute }, Inherit { flag: DelegationFnSigAttrs, factory: fn(Span, &hir::Attribute) -> hir::Attribute }, } const PARENT_ID: hir::ItemLocalId = hir::ItemLocalId::ZERO; -static ATTRIBUTES_ADDITIONS: &[AttributeAdditionInfo] = &[ - AttributeAdditionInfo { +static ATTRS_ADDITIONS: &[AttrAdditionInfo] = &[ + AttrAdditionInfo { equals: |a| matches!(a, hir::Attribute::Parsed(AttributeKind::MustUse { .. })), - kind: AttributeAdditionKind::Inherit { - factory: |span, original_attribute| { - let reason = match original_attribute { + kind: AttrAdditionKind::Inherit { + factory: |span, original_attr| { + let reason = match original_attr { hir::Attribute::Parsed(AttributeKind::MustUse { reason, .. }) => *reason, _ => None, }; @@ -91,14 +94,41 @@ static ATTRIBUTES_ADDITIONS: &[AttributeAdditionInfo] = &[ flag: DelegationFnSigAttrs::MUST_USE, }, }, - AttributeAdditionInfo { + AttrAdditionInfo { equals: |a| matches!(a, hir::Attribute::Parsed(AttributeKind::Inline(..))), - kind: AttributeAdditionKind::Default { + kind: AttrAdditionKind::Default { factory: |span| hir::Attribute::Parsed(AttributeKind::Inline(InlineAttr::Hint, span)), }, }, ]; +type DelegationIdsVec = SmallVec<[DefId; 1]>; + +// As delegations can now refer to another delegation, we have a delegation path +// of the following type: reuse (current delegation) <- reuse (delegee_id) <- ... <- reuse <- function (root_function_id). +// In its most basic and widely used form: reuse (current delegation) <- function (delegee_id, root_function_id) +struct DelegationIds { + path: DelegationIdsVec, +} + +impl DelegationIds { + fn new(path: DelegationIdsVec) -> Self { + assert!(!path.is_empty()); + Self { path } + } + + // Id of the first function in (non)local crate that is being reused + fn root_function_id(&self) -> DefId { + *self.path.last().expect("Ids vector can't be empty") + } + + // Id of the first definition which is being reused, + // can be either function, in this case `root_id == delegee_id`, or other delegation + fn delegee_id(&self) -> DefId { + *self.path.first().expect("Ids vector can't be empty") + } +} + impl<'hir> LoweringContext<'_, 'hir> { fn is_method(&self, def_id: DefId, span: Span) -> bool { match self.tcx.def_kind(def_id) { @@ -119,18 +149,39 @@ impl<'hir> LoweringContext<'_, 'hir> { &mut self, delegation: &Delegation, item_id: NodeId, - is_in_trait_impl: bool, ) -> DelegationResults<'hir> { let span = self.lower_span(delegation.path.segments.last().unwrap().ident.span); - let sig_id = self.get_delegation_sig_id(item_id, delegation.id, span, is_in_trait_impl); - match sig_id { - Ok(sig_id) => { - self.add_attributes_if_needed(span, sig_id); - let is_method = self.is_method(sig_id, span); - let (param_count, c_variadic) = self.param_count(sig_id); - let decl = self.lower_delegation_decl(sig_id, param_count, c_variadic, span); - let sig = self.lower_delegation_sig(sig_id, decl, span); + let ids = self.get_delegation_ids( + self.resolver.delegation_infos[&self.local_def_id(item_id)].resolution_node, + span, + ); + + match ids { + Ok(ids) => { + self.add_attrs_if_needed(span, &ids); + + let delegee_id = ids.delegee_id(); + let root_function_id = ids.root_function_id(); + + // `is_method` is used to choose the name of the first parameter (`self` or `arg0`), + // if the original function is not a method (without `self`), then it can not be added + // during chain of reuses, so we use `root_function_id` here + let is_method = self.is_method(root_function_id, span); + + // Here we use `root_function_id` as we can not get params information out of potential delegation reuse, + // we need a function to extract this information + let (param_count, c_variadic) = self.param_count(root_function_id); + + // Here we use `delegee_id`, as this id will then be used to calculate parent for generics + // inheritance, and we want this id to point on a delegee, not on the original + // function (see https://github.com/rust-lang/rust/issues/150152#issuecomment-3674834654) + let decl = self.lower_delegation_decl(delegee_id, param_count, c_variadic, span); + + // Here we pass `root_function_id` as we want to inherit signature (including consts, async) + // from the root function that started delegation + let sig = self.lower_delegation_sig(root_function_id, decl, span); + let body_id = self.lower_delegation_body(delegation, is_method, param_count, span); let ident = self.lower_ident(delegation.ident); let generics = self.lower_delegation_generics(span); @@ -140,36 +191,36 @@ impl<'hir> LoweringContext<'_, 'hir> { } } - fn add_attributes_if_needed(&mut self, span: Span, sig_id: DefId) { - let new_attributes = self.create_new_attributes( - ATTRIBUTES_ADDITIONS, - span, - sig_id, - self.attrs.get(&PARENT_ID), - ); + fn add_attrs_if_needed(&mut self, span: Span, ids: &DelegationIds) { + let new_attrs = + self.create_new_attrs(ATTRS_ADDITIONS, span, ids, self.attrs.get(&PARENT_ID)); - if new_attributes.is_empty() { + if new_attrs.is_empty() { return; } - let new_arena_allocated_attributes = match self.attrs.get(&PARENT_ID) { + let new_arena_allocated_attrs = match self.attrs.get(&PARENT_ID) { Some(existing_attrs) => self.arena.alloc_from_iter( - existing_attrs.iter().map(|a| a.clone()).chain(new_attributes.into_iter()), + existing_attrs.iter().map(|a| a.clone()).chain(new_attrs.into_iter()), ), - None => self.arena.alloc_from_iter(new_attributes.into_iter()), + None => self.arena.alloc_from_iter(new_attrs.into_iter()), }; - self.attrs.insert(PARENT_ID, new_arena_allocated_attributes); + self.attrs.insert(PARENT_ID, new_arena_allocated_attrs); } - fn create_new_attributes( + fn create_new_attrs( &self, - candidate_additions: &[AttributeAdditionInfo], + candidate_additions: &[AttrAdditionInfo], span: Span, - sig_id: DefId, + ids: &DelegationIds, existing_attrs: Option<&&[hir::Attribute]>, ) -> Vec { - let local_original_attributes = self.parse_local_original_attributes(sig_id); + let defs_orig_attrs = ids + .path + .iter() + .map(|def_id| (*def_id, self.parse_local_original_attrs(*def_id))) + .collect::>(); candidate_additions .iter() @@ -183,79 +234,120 @@ impl<'hir> LoweringContext<'_, 'hir> { } match addition_info.kind { - AttributeAdditionKind::Default { factory } => Some(factory(span)), - AttributeAdditionKind::Inherit { flag, factory } => { - let original_attribute = match sig_id.as_local() { - Some(local_id) => self - .resolver - .delegation_fn_sigs - .get(&local_id) - .is_some_and(|sig| sig.attrs_flags.contains(flag)) - .then(|| { - local_original_attributes - .as_ref() - .map(|attrs| { - attrs - .iter() - .find(|base_attr| (addition_info.equals)(base_attr)) - }) - .flatten() - }) - .flatten(), - None => self - .tcx - .get_all_attrs(sig_id) - .iter() - .find(|base_attr| (addition_info.equals)(base_attr)), - }; + AttrAdditionKind::Default { factory } => Some(factory(span)), + AttrAdditionKind::Inherit { flag, factory } => { + for (def_id, orig_attrs) in &defs_orig_attrs { + let original_attr = match def_id.as_local() { + Some(local_id) => self + .get_attrs(local_id) + .flags + .contains(flag) + .then(|| { + orig_attrs + .as_ref() + .map(|attrs| { + attrs.iter().find(|base_attr| { + (addition_info.equals)(base_attr) + }) + }) + .flatten() + }) + .flatten(), + None => self + .tcx + .get_all_attrs(*def_id) + .iter() + .find(|base_attr| (addition_info.equals)(base_attr)), + }; - original_attribute.map(|a| factory(span, a)) + if let Some(original_attr) = original_attr { + return Some(factory(span, original_attr)); + } + } + + None } } }) .collect::>() } - fn parse_local_original_attributes(&self, sig_id: DefId) -> Option> { - if let Some(local_id) = sig_id.as_local() - && let Some(info) = self.resolver.delegation_fn_sigs.get(&local_id) - && !info.to_inherit_attrs.is_empty() - { - Some(AttributeParser::parse_limited_all( - self.tcx.sess, - info.to_inherit_attrs.as_slice(), - None, - Target::Fn, - DUMMY_SP, - DUMMY_NODE_ID, - Some(self.tcx.features()), - ShouldEmit::Nothing, - )) + fn parse_local_original_attrs(&self, def_id: DefId) -> Option> { + if let Some(local_id) = def_id.as_local() { + let attrs = &self.get_attrs(local_id).to_inherit; + + if !attrs.is_empty() { + return Some(AttributeParser::parse_limited_all( + self.tcx.sess, + attrs, + None, + Target::Fn, + DUMMY_SP, + DUMMY_NODE_ID, + Some(self.tcx.features()), + ShouldEmit::Nothing, + )); + } + } + + None + } + + fn get_attrs(&self, local_id: LocalDefId) -> &DelegationAttrs { + // local_id can correspond either to a function or other delegation + if let Some(fn_sig) = self.resolver.delegation_fn_sigs.get(&local_id) { + &fn_sig.attrs } else { - None + &self.resolver.delegation_infos[&local_id].attrs } } - fn get_delegation_sig_id( + fn get_delegation_ids( &self, - item_id: NodeId, - path_id: NodeId, + mut node_id: NodeId, span: Span, - is_in_trait_impl: bool, - ) -> Result { - let sig_id = if is_in_trait_impl { item_id } else { path_id }; - self.get_resolution_id(sig_id, span) + ) -> Result { + let mut visited: FxHashSet = Default::default(); + let mut path: DelegationIdsVec = Default::default(); + + loop { + visited.insert(node_id); + + let Some(def_id) = self.get_resolution_id(node_id) else { + return Err(self.tcx.dcx().span_delayed_bug( + span, + format!( + "LoweringContext: couldn't resolve node {:?} in delegation item", + node_id + ), + )); + }; + + path.push(def_id); + + // If def_id is in local crate and it corresponds to another delegation + // it means that we refer to another delegation as a callee, so in order to obtain + // a signature DefId we obtain NodeId of the callee delegation and try to get signature from it. + if let Some(local_id) = def_id.as_local() + && let Some(delegation_info) = self.resolver.delegation_infos.get(&local_id) + { + node_id = delegation_info.resolution_node; + if visited.contains(&node_id) { + // We encountered a cycle in the resolution, or delegation callee refers to non-existent + // entity, in this case emit an error. + return Err(match visited.len() { + 1 => self.dcx().emit_err(UnresolvedDelegationCallee { span }), + _ => self.dcx().emit_err(CycleInDelegationSignatureResolution { span }), + }); + } + } else { + return Ok(DelegationIds::new(path)); + } + } } - fn get_resolution_id(&self, node_id: NodeId, span: Span) -> Result { - let def_id = - self.resolver.get_partial_res(node_id).and_then(|r| r.expect_full_res().opt_def_id()); - def_id.ok_or_else(|| { - self.tcx.dcx().span_delayed_bug( - span, - format!("LoweringContext: couldn't resolve node {:?} in delegation item", node_id), - ) - }) + fn get_resolution_id(&self, node_id: NodeId) -> Option { + self.resolver.get_partial_res(node_id).and_then(|r| r.expect_full_res().opt_def_id()) } fn lower_delegation_generics(&mut self, span: Span) -> &'hir hir::Generics<'hir> { @@ -269,16 +361,14 @@ impl<'hir> LoweringContext<'_, 'hir> { } // Function parameter count, including C variadic `...` if present. - fn param_count(&self, sig_id: DefId) -> (usize, bool /*c_variadic*/) { - if let Some(local_sig_id) = sig_id.as_local() { - // Map may be filled incorrectly due to recursive delegation. - // Error will be emitted later during HIR ty lowering. + fn param_count(&self, def_id: DefId) -> (usize, bool /*c_variadic*/) { + if let Some(local_sig_id) = def_id.as_local() { match self.resolver.delegation_fn_sigs.get(&local_sig_id) { Some(sig) => (sig.param_count, sig.c_variadic), None => (0, false), } } else { - let sig = self.tcx.fn_sig(sig_id).skip_binder().skip_binder(); + let sig = self.tcx.fn_sig(def_id).skip_binder().skip_binder(); (sig.inputs().len() + usize::from(sig.c_variadic), sig.c_variadic) } } @@ -328,7 +418,7 @@ impl<'hir> LoweringContext<'_, 'hir> { // We are not forwarding the attributes, as the delegation fn sigs are collected on the ast, // and here we need the hir attributes. let default_safety = - if sig.attrs_flags.contains(DelegationFnSigAttrs::TARGET_FEATURE) + if sig.attrs.flags.contains(DelegationFnSigAttrs::TARGET_FEATURE) || self.tcx.def_kind(parent) == DefKind::ForeignMod { hir::Safety::Unsafe @@ -489,8 +579,8 @@ impl<'hir> LoweringContext<'_, 'hir> { delegation.path.segments.iter().rev().skip(1).any(|segment| segment.args.is_some()); let call = if self - .get_resolution_id(delegation.id, span) - .and_then(|def_id| Ok(self.is_method(def_id, span))) + .get_resolution_id(delegation.id) + .map(|def_id| self.is_method(def_id, span)) .unwrap_or_default() && delegation.qself.is_none() && !has_generic_args diff --git a/compiler/rustc_ast_lowering/src/errors.rs b/compiler/rustc_ast_lowering/src/errors.rs index 83f3a976e83f..88e69e67d8a5 100644 --- a/compiler/rustc_ast_lowering/src/errors.rs +++ b/compiler/rustc_ast_lowering/src/errors.rs @@ -357,6 +357,8 @@ pub(crate) struct ArbitraryExpressionInPattern { pub span: Span, #[note(ast_lowering_pattern_from_macro_note)] pub pattern_from_macro_note: bool, + #[help(ast_lowering_const_block_in_pattern_help)] + pub const_block_in_pattern_help: bool, } #[derive(Diagnostic)] @@ -475,3 +477,17 @@ pub(crate) struct UnionWithDefault { #[primary_span] pub span: Span, } + +#[derive(Diagnostic)] +#[diag(ast_lowering_delegation_unresolved_callee)] +pub(crate) struct UnresolvedDelegationCallee { + #[primary_span] + pub span: Span, +} + +#[derive(Diagnostic)] +#[diag(ast_lowering_delegation_cycle_in_signature_resolution)] +pub(crate) struct CycleInDelegationSignatureResolution { + #[primary_span] + pub span: Span, +} diff --git a/compiler/rustc_ast_lowering/src/expr.rs b/compiler/rustc_ast_lowering/src/expr.rs index c8a311443a58..0502fd2873e9 100644 --- a/compiler/rustc_ast_lowering/src/expr.rs +++ b/compiler/rustc_ast_lowering/src/expr.rs @@ -114,7 +114,8 @@ impl<'hir> LoweringContext<'_, 'hir> { } ExprKind::Tup(elts) => hir::ExprKind::Tup(self.lower_exprs(elts)), ExprKind::Call(f, args) => { - if let Some(legacy_args) = self.resolver.legacy_const_generic_args(f) { + if let Some(legacy_args) = self.resolver.legacy_const_generic_args(f, self.tcx) + { self.lower_legacy_const_generics((**f).clone(), args.clone(), &legacy_args) } else { let f = self.lower_expr(f); diff --git a/compiler/rustc_ast_lowering/src/index.rs b/compiler/rustc_ast_lowering/src/index.rs index 8d7351d3a510..f6edcaa64dfe 100644 --- a/compiler/rustc_ast_lowering/src/index.rs +++ b/compiler/rustc_ast_lowering/src/index.rs @@ -281,6 +281,13 @@ impl<'a, 'hir> Visitor<'hir> for NodeCollector<'a, 'hir> { }); } + fn visit_const_arg_expr_field(&mut self, field: &'hir ConstArgExprField<'hir>) { + self.insert(field.span, field.hir_id, Node::ConstArgExprField(field)); + self.with_parent(field.hir_id, |this| { + intravisit::walk_const_arg_expr_field(this, field); + }) + } + fn visit_stmt(&mut self, stmt: &'hir Stmt<'hir>) { self.insert(stmt.span, stmt.hir_id, Node::Stmt(stmt)); diff --git a/compiler/rustc_ast_lowering/src/item.rs b/compiler/rustc_ast_lowering/src/item.rs index a68d63bf1464..bfce7c25b75d 100644 --- a/compiler/rustc_ast_lowering/src/item.rs +++ b/compiler/rustc_ast_lowering/src/item.rs @@ -541,7 +541,7 @@ impl<'hir> LoweringContext<'_, 'hir> { hir::ItemKind::Macro(ident, macro_def, macro_kinds) } ItemKind::Delegation(box delegation) => { - let delegation_results = self.lower_delegation(delegation, id, false); + let delegation_results = self.lower_delegation(delegation, id); hir::ItemKind::Fn { sig: delegation_results.sig, ident: delegation_results.ident, @@ -1026,7 +1026,7 @@ impl<'hir> LoweringContext<'_, 'hir> { (*ident, generics, kind, ty.is_some()) } AssocItemKind::Delegation(box delegation) => { - let delegation_results = self.lower_delegation(delegation, i.id, false); + let delegation_results = self.lower_delegation(delegation, i.id); let item_kind = hir::TraitItemKind::Fn( delegation_results.sig, hir::TraitFn::Provided(delegation_results.body_id), @@ -1196,7 +1196,7 @@ impl<'hir> LoweringContext<'_, 'hir> { ) } AssocItemKind::Delegation(box delegation) => { - let delegation_results = self.lower_delegation(delegation, i.id, is_in_trait_impl); + let delegation_results = self.lower_delegation(delegation, i.id); ( delegation.ident, ( diff --git a/compiler/rustc_ast_lowering/src/lib.rs b/compiler/rustc_ast_lowering/src/lib.rs index 10093da97fe3..416fef8e3af3 100644 --- a/compiler/rustc_ast_lowering/src/lib.rs +++ b/compiler/rustc_ast_lowering/src/lib.rs @@ -47,13 +47,14 @@ use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::sync::spawn; use rustc_data_structures::tagged_ptr::TaggedRef; use rustc_errors::{DiagArgFromDisplay, DiagCtxtHandle}; +use rustc_hir::attrs::AttributeKind; use rustc_hir::def::{DefKind, LifetimeRes, Namespace, PartialRes, PerNS, Res}; use rustc_hir::def_id::{CRATE_DEF_ID, LOCAL_CRATE, LocalDefId}; use rustc_hir::definitions::{DefPathData, DisambiguatorState}; use rustc_hir::lints::DelayedLint; use rustc_hir::{ self as hir, AngleBrackets, ConstArg, GenericArg, HirId, ItemLocalMap, LifetimeSource, - LifetimeSyntax, ParamName, Target, TraitCandidate, + LifetimeSyntax, ParamName, Target, TraitCandidate, find_attr, }; use rustc_index::{Idx, IndexSlice, IndexVec}; use rustc_macros::extension; @@ -236,29 +237,32 @@ impl SpanLowerer { #[extension(trait ResolverAstLoweringExt)] impl ResolverAstLowering { - fn legacy_const_generic_args(&self, expr: &Expr) -> Option> { - if let ExprKind::Path(None, path) = &expr.kind { - // Don't perform legacy const generics rewriting if the path already - // has generic arguments. - if path.segments.last().unwrap().args.is_some() { - return None; - } + fn legacy_const_generic_args(&self, expr: &Expr, tcx: TyCtxt<'_>) -> Option> { + let ExprKind::Path(None, path) = &expr.kind else { + return None; + }; - if let Res::Def(DefKind::Fn, def_id) = self.partial_res_map.get(&expr.id)?.full_res()? { - // We only support cross-crate argument rewriting. Uses - // within the same crate should be updated to use the new - // const generics style. - if def_id.is_local() { - return None; - } - - if let Some(v) = self.legacy_const_generic_args.get(&def_id) { - return v.clone(); - } - } + // Don't perform legacy const generics rewriting if the path already + // has generic arguments. + if path.segments.last().unwrap().args.is_some() { + return None; } - None + let def_id = self.partial_res_map.get(&expr.id)?.full_res()?.opt_def_id()?; + + // We only support cross-crate argument rewriting. Uses + // within the same crate should be updated to use the new + // const generics style. + if def_id.is_local() { + return None; + } + + find_attr!( + // we can use parsed attrs here since for other crates they're already available + tcx.get_all_attrs(def_id), + AttributeKind::RustcLegacyConstGenerics{fn_indexes,..} => fn_indexes + ) + .map(|fn_indexes| fn_indexes.iter().map(|(num, _)| *num).collect()) } fn get_partial_res(&self, id: NodeId) -> Option { @@ -2406,6 +2410,47 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { ConstArg { hir_id: self.next_id(), kind: hir::ConstArgKind::Path(qpath) } } + ExprKind::Struct(se) => { + let path = self.lower_qpath( + expr.id, + &se.qself, + &se.path, + // FIXME(mgca): we may want this to be `Optional` instead, but + // we would also need to make sure that HIR ty lowering errors + // when these paths wind up in signatures. + ParamMode::Explicit, + AllowReturnTypeNotation::No, + ImplTraitContext::Disallowed(ImplTraitPosition::Path), + None, + ); + + let fields = self.arena.alloc_from_iter(se.fields.iter().map(|f| { + let hir_id = self.lower_node_id(f.id); + // FIXME(mgca): This might result in lowering attributes that + // then go unused as the `Target::ExprField` is not actually + // corresponding to `Node::ExprField`. + self.lower_attrs(hir_id, &f.attrs, f.span, Target::ExprField); + + let expr = if let ExprKind::ConstBlock(anon_const) = &f.expr.kind { + let def_id = self.local_def_id(anon_const.id); + let def_kind = self.tcx.def_kind(def_id); + assert_eq!(DefKind::AnonConst, def_kind); + + self.lower_anon_const_to_const_arg_direct(anon_const) + } else { + self.lower_expr_to_const_arg_direct(&f.expr) + }; + + &*self.arena.alloc(hir::ConstArgExprField { + hir_id, + field: self.lower_ident(f.ident), + expr: self.arena.alloc(expr), + span: self.lower_span(f.span), + }) + })); + + ConstArg { hir_id: self.next_id(), kind: hir::ConstArgKind::Struct(path, fields) } + } ExprKind::Underscore => ConstArg { hir_id: self.lower_node_id(expr.id), kind: hir::ConstArgKind::Infer(expr.span, ()), diff --git a/compiler/rustc_ast_lowering/src/pat.rs b/compiler/rustc_ast_lowering/src/pat.rs index 3571fd652397..7a5d8d9847a1 100644 --- a/compiler/rustc_ast_lowering/src/pat.rs +++ b/compiler/rustc_ast_lowering/src/pat.rs @@ -399,7 +399,6 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { ExprKind::Lit(lit) => { hir::PatExprKind::Lit { lit: self.lower_lit(lit, span), negated: false } } - ExprKind::ConstBlock(c) => hir::PatExprKind::ConstBlock(self.lower_const_block(c)), ExprKind::IncludedBytes(byte_sym) => hir::PatExprKind::Lit { lit: respan(span, LitKind::ByteStr(*byte_sym, StrStyle::Cooked)), negated: false, @@ -419,10 +418,12 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> { hir::PatExprKind::Lit { lit: self.lower_lit(lit, span), negated: true } } _ => { + let is_const_block = matches!(expr.kind, ExprKind::ConstBlock(_)); let pattern_from_macro = expr.is_approximately_pattern(); let guar = self.dcx().emit_err(ArbitraryExpressionInPattern { span, pattern_from_macro_note: pattern_from_macro, + const_block_in_pattern_help: is_const_block, }); err(guar) } diff --git a/compiler/rustc_ast_pretty/src/pprust/mod.rs b/compiler/rustc_ast_pretty/src/pprust/mod.rs index a766e2006e59..74ed0405498d 100644 --- a/compiler/rustc_ast_pretty/src/pprust/mod.rs +++ b/compiler/rustc_ast_pretty/src/pprust/mod.rs @@ -33,6 +33,9 @@ pub fn where_bound_predicate_to_string(where_bound_predicate: &ast::WhereBoundPr State::new().where_bound_predicate_to_string(where_bound_predicate) } +/// # Panics +/// +/// Panics if `pat.kind` is `PatKind::Missing`. pub fn pat_to_string(pat: &ast::Pat) -> String { State::new().pat_to_string(pat) } diff --git a/compiler/rustc_attr_parsing/src/attributes/cfg_select.rs b/compiler/rustc_attr_parsing/src/attributes/cfg_select.rs index 0c0915558089..00a2d12106e7 100644 --- a/compiler/rustc_attr_parsing/src/attributes/cfg_select.rs +++ b/compiler/rustc_attr_parsing/src/attributes/cfg_select.rs @@ -28,6 +28,33 @@ pub struct CfgSelectBranches { pub unreachable: Vec<(CfgSelectPredicate, TokenStream, Span)>, } +impl CfgSelectBranches { + /// Removes the top-most branch for which `predicate` returns `true`, + /// or the wildcard if none of the reachable branches satisfied the predicate. + pub fn pop_first_match(&mut self, predicate: F) -> Option<(TokenStream, Span)> + where + F: Fn(&CfgEntry) -> bool, + { + for (index, (cfg, _, _)) in self.reachable.iter().enumerate() { + if predicate(cfg) { + let matched = self.reachable.remove(index); + return Some((matched.1, matched.2)); + } + } + + self.wildcard.take().map(|(_, tts, span)| (tts, span)) + } + + /// Consume this value and iterate over all the `TokenStream`s that it stores. + pub fn into_iter_tts(self) -> impl Iterator { + let it1 = self.reachable.into_iter().map(|(_, tts, span)| (tts, span)); + let it2 = self.wildcard.into_iter().map(|(_, tts, span)| (tts, span)); + let it3 = self.unreachable.into_iter().map(|(_, tts, span)| (tts, span)); + + it1.chain(it2).chain(it3) + } +} + pub fn parse_cfg_select( p: &mut Parser<'_>, sess: &Session, diff --git a/compiler/rustc_attr_parsing/src/attributes/cfi_encoding.rs b/compiler/rustc_attr_parsing/src/attributes/cfi_encoding.rs new file mode 100644 index 000000000000..df1e569743c0 --- /dev/null +++ b/compiler/rustc_attr_parsing/src/attributes/cfi_encoding.rs @@ -0,0 +1,33 @@ +use super::prelude::*; +pub(crate) struct CfiEncodingParser; +impl SingleAttributeParser for CfiEncodingParser { + const PATH: &[Symbol] = &[sym::cfi_encoding]; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowListWarnRest(&[ + Allow(Target::Struct), + Allow(Target::ForeignTy), + Allow(Target::Enum), + Allow(Target::Union), + ]); + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const TEMPLATE: AttributeTemplate = template!(NameValueStr: "encoding"); + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option { + let Some(name_value) = args.name_value() else { + cx.expected_name_value(cx.attr_span, Some(sym::cfi_encoding)); + return None; + }; + + let Some(value_str) = name_value.value_as_str() else { + cx.expected_string_literal(name_value.value_span, None); + return None; + }; + + if value_str.as_str().trim().is_empty() { + cx.expected_non_empty_string_literal(name_value.value_span); + return None; + } + + Some(AttributeKind::CfiEncoding { encoding: value_str }) + } +} diff --git a/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs b/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs index e59f70877c94..c68c66b27185 100644 --- a/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs +++ b/compiler/rustc_attr_parsing/src/attributes/codegen_attrs.rs @@ -690,6 +690,16 @@ impl SingleAttributeParser for SanitizeParser { } } +pub(crate) struct ThreadLocalParser; + +impl NoArgsAttributeParser for ThreadLocalParser { + const PATH: &[Symbol] = &[sym::thread_local]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::WarnButFutureError; + const ALLOWED_TARGETS: AllowedTargets = + AllowedTargets::AllowList(&[Allow(Target::Static), Allow(Target::ForeignStatic)]); + const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::ThreadLocal; +} + pub(crate) struct RustcPassIndirectlyInNonRusticAbisParser; impl NoArgsAttributeParser for RustcPassIndirectlyInNonRusticAbisParser { diff --git a/compiler/rustc_attr_parsing/src/attributes/mod.rs b/compiler/rustc_attr_parsing/src/attributes/mod.rs index 64f29a4729c3..c39c60ea7e39 100644 --- a/compiler/rustc_attr_parsing/src/attributes/mod.rs +++ b/compiler/rustc_attr_parsing/src/attributes/mod.rs @@ -33,6 +33,7 @@ pub(crate) mod allow_unstable; pub(crate) mod body; pub(crate) mod cfg; pub(crate) mod cfg_select; +pub(crate) mod cfi_encoding; pub(crate) mod codegen_attrs; pub(crate) mod confusables; pub(crate) mod crate_level; @@ -47,6 +48,7 @@ pub(crate) mod loop_match; pub(crate) mod macro_attrs; pub(crate) mod must_use; pub(crate) mod no_implicit_prelude; +pub(crate) mod no_link; pub(crate) mod non_exhaustive; pub(crate) mod path; pub(crate) mod pin_v2; diff --git a/compiler/rustc_attr_parsing/src/attributes/no_link.rs b/compiler/rustc_attr_parsing/src/attributes/no_link.rs new file mode 100644 index 000000000000..43cd1c5406e9 --- /dev/null +++ b/compiler/rustc_attr_parsing/src/attributes/no_link.rs @@ -0,0 +1,14 @@ +use super::prelude::*; + +pub(crate) struct NoLinkParser; +impl NoArgsAttributeParser for NoLinkParser { + const PATH: &[Symbol] = &[sym::no_link]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Warn; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::ExternCrate), + Warn(Target::Field), + Warn(Target::Arm), + Warn(Target::MacroDef), + ]); + const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::NoLink; +} diff --git a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs index 4be141651ae6..922a5bd297ac 100644 --- a/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs +++ b/compiler/rustc_attr_parsing/src/attributes/rustc_internal.rs @@ -1,3 +1,6 @@ +use rustc_ast::{LitIntType, LitKind, MetaItemLit}; +use rustc_session::errors; + use super::prelude::*; use super::util::parse_single_integer; use crate::session_diagnostics::RustcScalableVectorCountOutOfRange; @@ -11,6 +14,82 @@ impl NoArgsAttributeParser for RustcMainParser { const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcMain; } +pub(crate) struct RustcMustImplementOneOfParser; + +impl SingleAttributeParser for RustcMustImplementOneOfParser { + const PATH: &[Symbol] = &[sym::rustc_must_implement_one_of]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Trait)]); + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; + const TEMPLATE: AttributeTemplate = template!(List: &["function1, function2, ..."]); + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option { + let Some(list) = args.list() else { + cx.expected_list(cx.attr_span, args); + return None; + }; + + let mut fn_names = ThinVec::new(); + + let inputs: Vec<_> = list.mixed().collect(); + + if inputs.len() < 2 { + cx.expected_list_with_num_args_or_more(2, list.span); + return None; + } + + let mut errored = false; + for argument in inputs { + let Some(meta) = argument.meta_item() else { + cx.expected_identifier(argument.span()); + return None; + }; + + let Some(ident) = meta.ident() else { + cx.dcx().emit_err(errors::MustBeNameOfAssociatedFunction { span: meta.span() }); + errored = true; + continue; + }; + + fn_names.push(ident); + } + if errored { + return None; + } + + Some(AttributeKind::RustcMustImplementOneOf { attr_span: cx.attr_span, fn_names }) + } +} + +pub(crate) struct RustcNeverReturnsNullPointerParser; + +impl NoArgsAttributeParser for RustcNeverReturnsNullPointerParser { + const PATH: &[Symbol] = &[sym::rustc_never_returns_null_ptr]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + ]); + const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcNeverReturnsNullPointer; +} +pub(crate) struct RustcNoImplicitAutorefsParser; + +impl NoArgsAttributeParser for RustcNoImplicitAutorefsParser { + const PATH: &[Symbol] = &[sym::rustc_no_implicit_autorefs]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + ]); + + const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcNoImplicitAutorefs; +} + pub(crate) struct RustcLayoutScalarValidRangeStartParser; impl SingleAttributeParser for RustcLayoutScalarValidRangeStartParser { @@ -41,6 +120,129 @@ impl SingleAttributeParser for RustcLayoutScalarValidRangeEndParser } } +pub(crate) struct RustcLegacyConstGenericsParser; + +impl SingleAttributeParser for RustcLegacyConstGenericsParser { + const PATH: &[Symbol] = &[sym::rustc_legacy_const_generics]; + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Fn)]); + const TEMPLATE: AttributeTemplate = template!(List: &["N"]); + + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option { + let ArgParser::List(meta_items) = args else { + cx.expected_list(cx.attr_span, args); + return None; + }; + + let mut parsed_indexes = ThinVec::new(); + let mut errored = false; + + for possible_index in meta_items.mixed() { + if let MetaItemOrLitParser::Lit(MetaItemLit { + kind: LitKind::Int(index, LitIntType::Unsuffixed), + .. + }) = possible_index + { + parsed_indexes.push((index.0 as usize, possible_index.span())); + } else { + cx.expected_integer_literal(possible_index.span()); + errored = true; + } + } + if errored { + return None; + } else if parsed_indexes.is_empty() { + cx.expected_at_least_one_argument(args.span()?); + return None; + } + + Some(AttributeKind::RustcLegacyConstGenerics { + fn_indexes: parsed_indexes, + attr_span: cx.attr_span, + }) + } +} + +pub(crate) struct RustcLintDiagnosticsParser; + +impl NoArgsAttributeParser for RustcLintDiagnosticsParser { + const PATH: &[Symbol] = &[sym::rustc_lint_diagnostics]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + ]); + const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcLintDiagnostics; +} + +pub(crate) struct RustcLintOptDenyFieldAccessParser; + +impl SingleAttributeParser for RustcLintOptDenyFieldAccessParser { + const PATH: &[Symbol] = &[sym::rustc_lint_opt_deny_field_access]; + const ATTRIBUTE_ORDER: AttributeOrder = AttributeOrder::KeepInnermost; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Field)]); + const TEMPLATE: AttributeTemplate = template!(Word); + fn convert(cx: &mut AcceptContext<'_, '_, S>, args: &ArgParser) -> Option { + let Some(arg) = args.list().and_then(MetaItemListParser::single) else { + cx.expected_single_argument(cx.attr_span); + return None; + }; + + let MetaItemOrLitParser::Lit(MetaItemLit { kind: LitKind::Str(lint_message, _), .. }) = arg + else { + cx.expected_string_literal(arg.span(), arg.lit()); + return None; + }; + + Some(AttributeKind::RustcLintOptDenyFieldAccess { lint_message: *lint_message }) + } +} + +pub(crate) struct RustcLintOptTyParser; + +impl NoArgsAttributeParser for RustcLintOptTyParser { + const PATH: &[Symbol] = &[sym::rustc_lint_opt_ty]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[Allow(Target::Struct)]); + const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcLintOptTy; +} + +pub(crate) struct RustcLintQueryInstabilityParser; + +impl NoArgsAttributeParser for RustcLintQueryInstabilityParser { + const PATH: &[Symbol] = &[sym::rustc_lint_query_instability]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + ]); + const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcLintQueryInstability; +} + +pub(crate) struct RustcLintUntrackedQueryInformationParser; + +impl NoArgsAttributeParser for RustcLintUntrackedQueryInformationParser { + const PATH: &[Symbol] = &[sym::rustc_lint_untracked_query_information]; + const ON_DUPLICATE: OnDuplicate = OnDuplicate::Error; + const ALLOWED_TARGETS: AllowedTargets = AllowedTargets::AllowList(&[ + Allow(Target::Fn), + Allow(Target::Method(MethodKind::Inherent)), + Allow(Target::Method(MethodKind::Trait { body: false })), + Allow(Target::Method(MethodKind::Trait { body: true })), + Allow(Target::Method(MethodKind::TraitImpl)), + ]); + + const CREATE: fn(Span) -> AttributeKind = |_| AttributeKind::RustcLintUntrackedQueryInformation; +} + pub(crate) struct RustcObjectLifetimeDefaultParser; impl SingleAttributeParser for RustcObjectLifetimeDefaultParser { diff --git a/compiler/rustc_attr_parsing/src/context.rs b/compiler/rustc_attr_parsing/src/context.rs index 9c7006f84024..e67d089d3205 100644 --- a/compiler/rustc_attr_parsing/src/context.rs +++ b/compiler/rustc_attr_parsing/src/context.rs @@ -19,11 +19,12 @@ use crate::attributes::allow_unstable::{ AllowConstFnUnstableParser, AllowInternalUnstableParser, UnstableFeatureBoundParser, }; use crate::attributes::body::CoroutineParser; +use crate::attributes::cfi_encoding::CfiEncodingParser; use crate::attributes::codegen_attrs::{ ColdParser, CoverageParser, EiiExternItemParser, ExportNameParser, ForceTargetFeatureParser, NakedParser, NoMangleParser, ObjcClassParser, ObjcSelectorParser, OptimizeParser, RustcPassIndirectlyInNonRusticAbisParser, SanitizeParser, TargetFeatureParser, - TrackCallerParser, UsedParser, + ThreadLocalParser, TrackCallerParser, UsedParser, }; use crate::attributes::confusables::ConfusablesParser; use crate::attributes::crate_level::{ @@ -50,6 +51,7 @@ use crate::attributes::macro_attrs::{ }; use crate::attributes::must_use::MustUseParser; use crate::attributes::no_implicit_prelude::NoImplicitPreludeParser; +use crate::attributes::no_link::NoLinkParser; use crate::attributes::non_exhaustive::NonExhaustiveParser; use crate::attributes::path::PathParser as PathAttributeParser; use crate::attributes::pin_v2::PinV2Parser; @@ -59,7 +61,11 @@ use crate::attributes::proc_macro_attrs::{ use crate::attributes::prototype::CustomMirParser; use crate::attributes::repr::{AlignParser, AlignStaticParser, ReprParser}; use crate::attributes::rustc_internal::{ - RustcLayoutScalarValidRangeEndParser, RustcLayoutScalarValidRangeStartParser, RustcMainParser, + RustcLayoutScalarValidRangeEndParser, RustcLayoutScalarValidRangeStartParser, + RustcLegacyConstGenericsParser, RustcLintDiagnosticsParser, RustcLintOptDenyFieldAccessParser, + RustcLintOptTyParser, RustcLintQueryInstabilityParser, + RustcLintUntrackedQueryInformationParser, RustcMainParser, RustcMustImplementOneOfParser, + RustcNeverReturnsNullPointerParser, RustcNoImplicitAutorefsParser, RustcObjectLifetimeDefaultParser, RustcScalableVectorParser, RustcSimdMonomorphizeLaneLimitParser, }; @@ -183,6 +189,7 @@ attribute_parsers!( // tidy-alphabetical-end // tidy-alphabetical-start + Single, Single, Single, Single, @@ -209,6 +216,9 @@ attribute_parsers!( Single, Single, Single, + Single, + Single, + Single, Single, Single, Single, @@ -240,6 +250,7 @@ attribute_parsers!( Single>, Single>, Single>, + Single>, Single>, Single>, Single>, @@ -251,11 +262,18 @@ attribute_parsers!( Single>, Single>, Single>, + Single>, + Single>, + Single>, + Single>, Single>, + Single>, + Single>, Single>, Single>, Single>, Single>, + Single>, Single>, Single>, Single>, @@ -476,6 +494,17 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { self.emit_parse_error(span, AttributeParseErrorReason::ExpectedList) } + pub(crate) fn expected_list_with_num_args_or_more( + &self, + args: usize, + span: Span, + ) -> ErrorGuaranteed { + self.emit_parse_error( + span, + AttributeParseErrorReason::ExpectedListWithNumArgsOrMore { args }, + ) + } + pub(crate) fn expected_list_or_no_args(&self, span: Span) -> ErrorGuaranteed { self.emit_parse_error(span, AttributeParseErrorReason::ExpectedListOrNoArgs) } @@ -484,6 +513,10 @@ impl<'f, 'sess: 'f, S: Stage> AcceptContext<'f, 'sess, S> { self.emit_parse_error(span, AttributeParseErrorReason::ExpectedNameValueOrNoArgs) } + pub(crate) fn expected_non_empty_string_literal(&self, span: Span) -> ErrorGuaranteed { + self.emit_parse_error(span, AttributeParseErrorReason::ExpectedNonEmptyStringLiteral) + } + pub(crate) fn expected_no_args(&self, span: Span) -> ErrorGuaranteed { self.emit_parse_error(span, AttributeParseErrorReason::ExpectedNoArgs) } diff --git a/compiler/rustc_attr_parsing/src/parser.rs b/compiler/rustc_attr_parsing/src/parser.rs index 9551744d5ec5..474f848038d1 100644 --- a/compiler/rustc_attr_parsing/src/parser.rs +++ b/compiler/rustc_attr_parsing/src/parser.rs @@ -8,7 +8,9 @@ use std::fmt::{Debug, Display}; use rustc_ast::token::{self, Delimiter, MetaVarKind}; use rustc_ast::tokenstream::TokenStream; -use rustc_ast::{AttrArgs, Expr, ExprKind, LitKind, MetaItemLit, Path, StmtKind, UnOp}; +use rustc_ast::{ + AttrArgs, Expr, ExprKind, LitKind, MetaItemLit, Path, PathSegment, StmtKind, UnOp, +}; use rustc_ast_pretty::pprust; use rustc_errors::{Diag, PResult}; use rustc_hir::{self as hir, AttrPath}; @@ -256,6 +258,11 @@ impl Debug for MetaItemParser { } impl MetaItemParser { + /// For a single-segment meta item, returns its name; otherwise, returns `None`. + pub fn ident(&self) -> Option { + if let [PathSegment { ident, .. }] = self.path.0.segments[..] { Some(ident) } else { None } + } + pub fn span(&self) -> Span { if let Some(other) = self.args.span() { self.path.borrow().span().with_hi(other.hi()) diff --git a/compiler/rustc_attr_parsing/src/session_diagnostics.rs b/compiler/rustc_attr_parsing/src/session_diagnostics.rs index 73b65193fd34..3e523082d616 100644 --- a/compiler/rustc_attr_parsing/src/session_diagnostics.rs +++ b/compiler/rustc_attr_parsing/src/session_diagnostics.rs @@ -520,7 +520,11 @@ pub(crate) enum AttributeParseErrorReason<'a> { ExpectedSingleArgument, ExpectedList, ExpectedListOrNoArgs, + ExpectedListWithNumArgsOrMore { + args: usize, + }, ExpectedNameValueOrNoArgs, + ExpectedNonEmptyStringLiteral, UnexpectedLiteral, ExpectedNameValue(Option), DuplicateKey(Symbol), @@ -596,9 +600,15 @@ impl<'a, G: EmissionGuarantee> Diagnostic<'a, G> for AttributeParseError<'_> { AttributeParseErrorReason::ExpectedListOrNoArgs => { diag.span_label(self.span, "expected a list or no arguments here"); } + AttributeParseErrorReason::ExpectedListWithNumArgsOrMore { args } => { + diag.span_label(self.span, format!("expected {args} or more items")); + } AttributeParseErrorReason::ExpectedNameValueOrNoArgs => { diag.span_label(self.span, "didn't expect a list here"); } + AttributeParseErrorReason::ExpectedNonEmptyStringLiteral => { + diag.span_label(self.span, "string is not allowed to be empty"); + } AttributeParseErrorReason::DuplicateKey(key) => { diag.span_label(self.span, format!("found `{key}` used as a key more than once")); diag.code(E0538); diff --git a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs index 8e18bf557758..743a28822eb9 100644 --- a/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs +++ b/compiler/rustc_borrowck/src/diagnostics/explain_borrow.rs @@ -764,6 +764,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> { { // Just point to the function, to reduce the chance of overlapping spans. let function_span = match func { + Operand::RuntimeChecks(_) => span, Operand::Constant(c) => c.span, Operand::Copy(place) | Operand::Move(place) => { if let Some(l) = place.as_local() { @@ -809,6 +810,7 @@ impl<'tcx> MirBorrowckCtxt<'_, '_, 'tcx> { { // Just point to the function, to reduce the chance of overlapping spans. let function_span = match func { + Operand::RuntimeChecks(_) => span, Operand::Constant(c) => c.span, Operand::Copy(place) | Operand::Move(place) => { if let Some(l) = place.as_local() { diff --git a/compiler/rustc_borrowck/src/lib.rs b/compiler/rustc_borrowck/src/lib.rs index 8d61ffde116c..91defbad0a0e 100644 --- a/compiler/rustc_borrowck/src/lib.rs +++ b/compiler/rustc_borrowck/src/lib.rs @@ -1,4 +1,4 @@ -//! This query borrow-checks the MIR to (further) ensure it is not broken. +//! This crate implemens MIR typeck and MIR borrowck. // tidy-alphabetical-start #![allow(internal_features)] @@ -111,9 +111,9 @@ pub fn provide(providers: &mut Providers) { *providers = Providers { mir_borrowck, ..*providers }; } -/// Provider for `query mir_borrowck`. Similar to `typeck`, this must -/// only be called for typeck roots which will then borrowck all -/// nested bodies as well. +/// Provider for `query mir_borrowck`. Unlike `typeck`, this must +/// only be called for typeck roots which *similar* to `typeck` will +/// then borrowck all nested bodies as well. fn mir_borrowck( tcx: TyCtxt<'_>, def: LocalDefId, @@ -1559,10 +1559,6 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { self.consume_operand(location, (operand2, span), state); } - Rvalue::NullaryOp(_op) => { - // nullary ops take no dynamic input; no borrowck effect. - } - Rvalue::Aggregate(aggregate_kind, operands) => { // We need to report back the list of mutable upvars that were // moved into the closure and subsequently used by the closure, @@ -1699,7 +1695,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { _ => propagate_closure_used_mut_place(self, place), } } - Operand::Constant(..) => {} + Operand::Constant(..) | Operand::RuntimeChecks(_) => {} } } @@ -1750,7 +1746,7 @@ impl<'a, 'tcx> MirBorrowckCtxt<'a, '_, 'tcx> { state, ); } - Operand::Constant(_) => {} + Operand::Constant(_) | Operand::RuntimeChecks(_) => {} } } diff --git a/compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs b/compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs index de479a7d74c8..d2eae2c7e65a 100644 --- a/compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs +++ b/compiler/rustc_borrowck/src/polonius/legacy/loan_invalidations.rs @@ -247,7 +247,7 @@ impl<'a, 'tcx> LoanInvalidationsGenerator<'a, 'tcx> { LocalMutationIsAllowed::Yes, ); } - Operand::Constant(_) => {} + Operand::Constant(_) | Operand::RuntimeChecks(_) => {} } } @@ -314,8 +314,6 @@ impl<'a, 'tcx> LoanInvalidationsGenerator<'a, 'tcx> { self.consume_operand(location, operand2); } - Rvalue::NullaryOp(_op) => {} - Rvalue::Aggregate(_, operands) => { for operand in operands { self.consume_operand(location, operand); diff --git a/compiler/rustc_borrowck/src/region_infer/mod.rs b/compiler/rustc_borrowck/src/region_infer/mod.rs index 80f1098de85e..6ed70b39c5b7 100644 --- a/compiler/rustc_borrowck/src/region_infer/mod.rs +++ b/compiler/rustc_borrowck/src/region_infer/mod.rs @@ -1275,29 +1275,81 @@ impl<'tcx> RegionInferenceContext<'tcx> { shorter_fr: RegionVid, propagated_outlives_requirements: &mut Option<&mut Vec>>, ) -> RegionRelationCheckResult { - if let Some(propagated_outlives_requirements) = propagated_outlives_requirements - // Shrink `longer_fr` until we find a non-local region (if we do). - // We'll call it `fr-` -- it's ever so slightly smaller than + if let Some(propagated_outlives_requirements) = propagated_outlives_requirements { + // Shrink `longer_fr` until we find some non-local regions. + // We'll call them `longer_fr-` -- they are ever so slightly smaller than // `longer_fr`. - && let Some(fr_minus) = self.universal_region_relations.non_local_lower_bound(longer_fr) - { - debug!("try_propagate_universal_region_error: fr_minus={:?}", fr_minus); + let longer_fr_minus = self.universal_region_relations.non_local_lower_bounds(longer_fr); + + debug!("try_propagate_universal_region_error: fr_minus={:?}", longer_fr_minus); + + // If we don't find a any non-local regions, we should error out as there is nothing + // to propagate. + if longer_fr_minus.is_empty() { + return RegionRelationCheckResult::Error; + } let blame_constraint = self .best_blame_constraint(longer_fr, NllRegionVariableOrigin::FreeRegion, shorter_fr) .0; - // Grow `shorter_fr` until we find some non-local regions. (We - // always will.) We'll call them `shorter_fr+` -- they're ever - // so slightly larger than `shorter_fr`. + // Grow `shorter_fr` until we find some non-local regions. + // We will always find at least one: `'static`. We'll call + // them `shorter_fr+` -- they're ever so slightly larger + // than `shorter_fr`. let shorter_fr_plus = self.universal_region_relations.non_local_upper_bounds(shorter_fr); debug!("try_propagate_universal_region_error: shorter_fr_plus={:?}", shorter_fr_plus); - for fr in shorter_fr_plus { - // Push the constraint `fr-: shorter_fr+` + + // We then create constraints `longer_fr-: shorter_fr+` that may or may not + // be propagated (see below). + let mut constraints = vec![]; + for fr_minus in longer_fr_minus { + for shorter_fr_plus in &shorter_fr_plus { + constraints.push((fr_minus, *shorter_fr_plus)); + } + } + + // We only need to propagate at least one of the constraints for + // soundness. However, we want to avoid arbitrary choices here + // and currently don't support returning OR constraints. + // + // If any of the `shorter_fr+` regions are already outlived by `longer_fr-`, + // we propagate only those. + // + // Consider this example (`'b: 'a` == `a -> b`), where we try to propagate `'d: 'a`: + // a --> b --> d + // \ + // \-> c + // Here, `shorter_fr+` of `'a` == `['b, 'c]`. + // Propagating `'d: 'b` is correct and should occur; `'d: 'c` is redundant because of + // `'d: 'b` and could reject valid code. + // + // So we filter the constraints to regions already outlived by `longer_fr-`, but if + // the filter yields an empty set, we fall back to the original one. + let subset: Vec<_> = constraints + .iter() + .filter(|&&(fr_minus, shorter_fr_plus)| { + self.eval_outlives(fr_minus, shorter_fr_plus) + }) + .copied() + .collect(); + let propagated_constraints = if subset.is_empty() { constraints } else { subset }; + debug!( + "try_propagate_universal_region_error: constraints={:?}", + propagated_constraints + ); + + assert!( + !propagated_constraints.is_empty(), + "Expected at least one constraint to propagate here" + ); + + for (fr_minus, fr_plus) in propagated_constraints { + // Push the constraint `long_fr-: shorter_fr+` propagated_outlives_requirements.push(ClosureOutlivesRequirement { subject: ClosureOutlivesSubject::Region(fr_minus), - outlived_free_region: fr, + outlived_free_region: fr_plus, blame_span: blame_constraint.cause.span, category: blame_constraint.category, }); diff --git a/compiler/rustc_borrowck/src/root_cx.rs b/compiler/rustc_borrowck/src/root_cx.rs index c3ff8cba9472..4d42055df168 100644 --- a/compiler/rustc_borrowck/src/root_cx.rs +++ b/compiler/rustc_borrowck/src/root_cx.rs @@ -255,7 +255,7 @@ impl<'tcx> BorrowCheckRootCtxt<'tcx> { } // We now apply the closure requirements of nested bodies modulo - // regions. In case a body does not depend on opaque types, we + // opaques. In case a body does not depend on opaque types, we // eagerly check its region constraints and use the final closure // requirements. // diff --git a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs index d27a73535bab..279625cb87c9 100644 --- a/compiler/rustc_borrowck/src/type_check/free_region_relations.rs +++ b/compiler/rustc_borrowck/src/type_check/free_region_relations.rs @@ -94,28 +94,10 @@ impl UniversalRegionRelations<'_> { /// words, returns the largest (*) known region `fr1` that (a) is /// outlived by `fr` and (b) is not local. /// - /// (*) If there are multiple competing choices, we pick the "postdominating" - /// one. See `TransitiveRelation::postdom_upper_bound` for details. - pub(crate) fn non_local_lower_bound(&self, fr: RegionVid) -> Option { + /// (*) If there are multiple competing choices, we return all of them. + pub(crate) fn non_local_lower_bounds(&self, fr: RegionVid) -> Vec { debug!("non_local_lower_bound(fr={:?})", fr); - let lower_bounds = self.non_local_bounds(&self.outlives, fr); - - // In case we find more than one, reduce to one for - // convenience. This is to prevent us from generating more - // complex constraints, but it will cause spurious errors. - let post_dom = self.outlives.mutual_immediate_postdominator(lower_bounds); - - debug!("non_local_bound: post_dom={:?}", post_dom); - - post_dom.and_then(|post_dom| { - // If the mutual immediate postdom is not local, then - // there is no non-local result we can return. - if !self.universal_regions.is_local_free_region(post_dom) { - Some(post_dom) - } else { - None - } - }) + self.non_local_bounds(&self.outlives, fr) } /// Helper for `non_local_upper_bounds` and `non_local_lower_bounds`. diff --git a/compiler/rustc_borrowck/src/type_check/mod.rs b/compiler/rustc_borrowck/src/type_check/mod.rs index 220de046293a..097416b4f280 100644 --- a/compiler/rustc_borrowck/src/type_check/mod.rs +++ b/compiler/rustc_borrowck/src/type_check/mod.rs @@ -1023,7 +1023,7 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { // element, so we require the `Copy` trait. if len.try_to_target_usize(tcx).is_none_or(|len| len > 1) { match operand { - Operand::Copy(..) | Operand::Constant(..) => { + Operand::Copy(..) | Operand::Constant(..) | Operand::RuntimeChecks(_) => { // These are always okay: direct use of a const, or a value that can // evidently be copied. } @@ -1046,8 +1046,6 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { } } - &Rvalue::NullaryOp(NullOp::RuntimeChecks(_)) => {} - Rvalue::ShallowInitBox(_operand, ty) => { let trait_ref = ty::TraitRef::new(tcx, tcx.require_lang_item(LangItem::Sized, span), [*ty]); @@ -2276,7 +2274,6 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> { | Rvalue::Cast(..) | Rvalue::ShallowInitBox(..) | Rvalue::BinaryOp(..) - | Rvalue::NullaryOp(..) | Rvalue::CopyForDeref(..) | Rvalue::UnaryOp(..) | Rvalue::Discriminant(..) diff --git a/compiler/rustc_borrowck/src/universal_regions.rs b/compiler/rustc_borrowck/src/universal_regions.rs index aee1eb94dc81..a0d5e1ce780d 100644 --- a/compiler/rustc_borrowck/src/universal_regions.rs +++ b/compiler/rustc_borrowck/src/universal_regions.rs @@ -1,13 +1,12 @@ //! Code to extract the universally quantified regions declared on a -//! function and the relationships between them. For example: +//! function. For example: //! //! ``` //! fn foo<'a, 'b, 'c: 'b>() { } //! ``` //! //! here we would return a map assigning each of `{'a, 'b, 'c}` -//! to an index, as well as the `FreeRegionMap` which can compute -//! relationships between them. +//! to an index. //! //! The code in this file doesn't *do anything* with those results; it //! just returns them for other code to use. @@ -271,8 +270,7 @@ impl<'tcx> UniversalRegions<'tcx> { /// Creates a new and fully initialized `UniversalRegions` that /// contains indices for all the free regions found in the given /// MIR -- that is, all the regions that appear in the function's - /// signature. This will also compute the relationships that are - /// known between those regions. + /// signature. pub(crate) fn new(infcx: &BorrowckInferCtxt<'tcx>, mir_def: LocalDefId) -> Self { UniversalRegionsBuilder { infcx, mir_def }.build() } @@ -648,17 +646,14 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> { BodyOwnerKind::Const { .. } | BodyOwnerKind::Static(..) => { let identity_args = GenericArgs::identity_for_item(tcx, typeck_root_def_id); - if self.mir_def.to_def_id() == typeck_root_def_id - // Do not ICE when checking default_field_values consts with lifetimes (#135649) - && DefKind::Field != tcx.def_kind(tcx.parent(typeck_root_def_id)) - { + if self.mir_def.to_def_id() == typeck_root_def_id { let args = self.infcx.replace_free_regions_with_nll_infer_vars( NllRegionVariableOrigin::FreeRegion, identity_args, ); DefiningTy::Const(self.mir_def.to_def_id(), args) } else { - // FIXME this line creates a dependency between borrowck and typeck. + // FIXME: this line creates a query dependency between borrowck and typeck. // // This is required for `AscribeUserType` canonical query, which will call // `type_of(inline_const_def_id)`. That `type_of` would inject erased lifetimes @@ -699,30 +694,14 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> { let tcx = self.infcx.tcx; let typeck_root_def_id = tcx.typeck_root_def_id(self.mir_def.to_def_id()); let identity_args = GenericArgs::identity_for_item(tcx, typeck_root_def_id); - let fr_args = match defining_ty { - DefiningTy::Closure(_, args) - | DefiningTy::CoroutineClosure(_, args) - | DefiningTy::Coroutine(_, args) - | DefiningTy::InlineConst(_, args) => { - // In the case of closures, we rely on the fact that - // the first N elements in the ClosureArgs are - // inherited from the `typeck_root_def_id`. - // Therefore, when we zip together (below) with - // `identity_args`, we will get only those regions - // that correspond to early-bound regions declared on - // the `typeck_root_def_id`. - assert!(args.len() >= identity_args.len()); - assert_eq!(args.regions().count(), identity_args.regions().count()); - args - } - - DefiningTy::FnDef(_, args) | DefiningTy::Const(_, args) => args, - - DefiningTy::GlobalAsm(_) => ty::List::empty(), - }; + let renumbered_args = defining_ty.args(); let global_mapping = iter::once((tcx.lifetimes.re_static, fr_static)); - let arg_mapping = iter::zip(identity_args.regions(), fr_args.regions().map(|r| r.as_var())); + // This relies on typeck roots being generics_of parents with their + // parameters at the start of nested bodies' generics. + assert!(renumbered_args.len() >= identity_args.len()); + let arg_mapping = + iter::zip(identity_args.regions(), renumbered_args.regions().map(|r| r.as_var())); UniversalRegionIndices { indices: global_mapping.chain(arg_mapping).collect(), @@ -862,8 +841,8 @@ impl<'cx, 'tcx> UniversalRegionsBuilder<'cx, 'tcx> { }; // FIXME(#129952): We probably want a more principled approach here. - if let Err(terr) = inputs_and_output.skip_binder().error_reported() { - self.infcx.set_tainted_by_errors(terr); + if let Err(e) = inputs_and_output.error_reported() { + self.infcx.set_tainted_by_errors(e); } inputs_and_output diff --git a/compiler/rustc_builtin_macros/messages.ftl b/compiler/rustc_builtin_macros/messages.ftl index 542f34d9d831..f0f6f2dcf82c 100644 --- a/compiler/rustc_builtin_macros/messages.ftl +++ b/compiler/rustc_builtin_macros/messages.ftl @@ -55,7 +55,6 @@ builtin_macros_assert_requires_expression = macro requires an expression as an a builtin_macros_autodiff = autodiff must be applied to function builtin_macros_autodiff_missing_config = autodiff requires at least a name and mode builtin_macros_autodiff_mode_activity = {$act} can not be used in {$mode} Mode -builtin_macros_autodiff_not_build = this rustc version does not support autodiff builtin_macros_autodiff_number_activities = expected {$expected} activities, but found {$found} builtin_macros_autodiff_ret_activity = invalid return activity {$act} in {$mode} Mode builtin_macros_autodiff_ty_activity = {$act} can not be used for this type diff --git a/compiler/rustc_builtin_macros/src/autodiff.rs b/compiler/rustc_builtin_macros/src/autodiff.rs index 6bf985fcc9f0..39abb66df30c 100644 --- a/compiler/rustc_builtin_macros/src/autodiff.rs +++ b/compiler/rustc_builtin_macros/src/autodiff.rs @@ -209,11 +209,6 @@ mod llvm_enzyme { mut item: Annotatable, mode: DiffMode, ) -> Vec { - // FIXME(bjorn3) maybe have the backend directly tell if autodiff is supported? - if cfg!(not(feature = "llvm_enzyme")) { - ecx.sess.dcx().emit_err(errors::AutoDiffSupportNotBuild { span: meta_item.span }); - return vec![item]; - } let dcx = ecx.sess.dcx(); // first get information about the annotable item: visibility, signature, name and generic diff --git a/compiler/rustc_builtin_macros/src/cfg_select.rs b/compiler/rustc_builtin_macros/src/cfg_select.rs index dc8077b2a1ff..f11190b28105 100644 --- a/compiler/rustc_builtin_macros/src/cfg_select.rs +++ b/compiler/rustc_builtin_macros/src/cfg_select.rs @@ -1,22 +1,65 @@ use rustc_ast::tokenstream::TokenStream; +use rustc_ast::{Expr, ast}; use rustc_attr_parsing as attr; use rustc_attr_parsing::{ CfgSelectBranches, CfgSelectPredicate, EvalConfigResult, parse_cfg_select, }; -use rustc_expand::base::{DummyResult, ExpandResult, ExtCtxt, MacroExpanderResult}; +use rustc_expand::base::{DummyResult, ExpandResult, ExtCtxt, MacResult, MacroExpanderResult}; use rustc_span::{Ident, Span, sym}; +use smallvec::SmallVec; use crate::errors::{CfgSelectNoMatches, CfgSelectUnreachable}; -/// Selects the first arm whose predicate evaluates to true. -fn select_arm(ecx: &ExtCtxt<'_>, branches: CfgSelectBranches) -> Option<(TokenStream, Span)> { - for (cfg, tt, arm_span) in branches.reachable { - if let EvalConfigResult::True = attr::eval_config_entry(&ecx.sess, &cfg) { - return Some((tt, arm_span)); - } - } +/// This intermediate structure is used to emit parse errors for the branches that are not chosen. +/// The `MacResult` instance below parses all branches, emitting any errors it encounters, but only +/// keeps the parse result for the selected branch. +struct CfgSelectResult<'cx, 'sess> { + ecx: &'cx mut ExtCtxt<'sess>, + site_span: Span, + selected_tts: TokenStream, + selected_span: Span, + other_branches: CfgSelectBranches, +} - branches.wildcard.map(|(_, tt, span)| (tt, span)) +fn tts_to_mac_result<'cx, 'sess>( + ecx: &'cx mut ExtCtxt<'sess>, + site_span: Span, + tts: TokenStream, + span: Span, +) -> Box { + match ExpandResult::from_tts(ecx, tts, site_span, span, Ident::with_dummy_span(sym::cfg_select)) + { + ExpandResult::Ready(x) => x, + _ => unreachable!("from_tts always returns Ready"), + } +} + +macro_rules! forward_to_parser_any_macro { + ($method_name:ident, $ret_ty:ty) => { + fn $method_name(self: Box) -> Option<$ret_ty> { + let CfgSelectResult { ecx, site_span, selected_tts, selected_span, .. } = *self; + + for (tts, span) in self.other_branches.into_iter_tts() { + let _ = tts_to_mac_result(ecx, site_span, tts, span).$method_name(); + } + + tts_to_mac_result(ecx, site_span, selected_tts, selected_span).$method_name() + } + }; +} + +impl<'cx, 'sess> MacResult for CfgSelectResult<'cx, 'sess> { + forward_to_parser_any_macro!(make_expr, Box); + forward_to_parser_any_macro!(make_stmts, SmallVec<[ast::Stmt; 1]>); + forward_to_parser_any_macro!(make_items, SmallVec<[Box; 1]>); + + forward_to_parser_any_macro!(make_impl_items, SmallVec<[Box; 1]>); + forward_to_parser_any_macro!(make_trait_impl_items, SmallVec<[Box; 1]>); + forward_to_parser_any_macro!(make_trait_items, SmallVec<[Box; 1]>); + forward_to_parser_any_macro!(make_foreign_items, SmallVec<[Box; 1]>); + + forward_to_parser_any_macro!(make_ty, Box); + forward_to_parser_any_macro!(make_pat, Box); } pub(super) fn expand_cfg_select<'cx>( @@ -31,7 +74,7 @@ pub(super) fn expand_cfg_select<'cx>( Some(ecx.ecfg.features), ecx.current_expansion.lint_node_id, ) { - Ok(branches) => { + Ok(mut branches) => { if let Some((underscore, _, _)) = branches.wildcard { // Warn for every unreachable predicate. We store the fully parsed branch for rustfmt. for (predicate, _, _) in &branches.unreachable { @@ -44,14 +87,17 @@ pub(super) fn expand_cfg_select<'cx>( } } - if let Some((tts, arm_span)) = select_arm(ecx, branches) { - return ExpandResult::from_tts( + if let Some((selected_tts, selected_span)) = branches.pop_first_match(|cfg| { + matches!(attr::eval_config_entry(&ecx.sess, cfg), EvalConfigResult::True) + }) { + let mac = CfgSelectResult { ecx, - tts, - sp, - arm_span, - Ident::with_dummy_span(sym::cfg_select), - ); + selected_tts, + selected_span, + other_branches: branches, + site_span: sp, + }; + return ExpandResult::Ready(Box::new(mac)); } else { // Emit a compiler error when none of the predicates matched. let guar = ecx.dcx().emit_err(CfgSelectNoMatches { span: sp }); diff --git a/compiler/rustc_builtin_macros/src/eii.rs b/compiler/rustc_builtin_macros/src/eii.rs index b97cb1daec53..9049639925dd 100644 --- a/compiler/rustc_builtin_macros/src/eii.rs +++ b/compiler/rustc_builtin_macros/src/eii.rs @@ -1,11 +1,12 @@ use rustc_ast::token::{Delimiter, TokenKind}; use rustc_ast::tokenstream::{DelimSpacing, DelimSpan, Spacing, TokenStream, TokenTree}; use rustc_ast::{ - DUMMY_NODE_ID, EiiExternTarget, EiiImpl, ItemKind, Stmt, StmtKind, ast, token, tokenstream, + Attribute, DUMMY_NODE_ID, EiiExternTarget, EiiImpl, ItemKind, MetaItem, Path, Stmt, StmtKind, + Visibility, ast, }; use rustc_ast_pretty::pprust::path_to_string; use rustc_expand::base::{Annotatable, ExtCtxt}; -use rustc_span::{Ident, Span, kw, sym}; +use rustc_span::{ErrorGuaranteed, Ident, Span, kw, sym}; use thin_vec::{ThinVec, thin_vec}; use crate::errors::{ @@ -52,138 +53,238 @@ pub(crate) fn unsafe_eii( fn eii_( ecx: &mut ExtCtxt<'_>, - span: Span, + eii_attr_span: Span, meta_item: &ast::MetaItem, item: Annotatable, impl_unsafe: bool, ) -> Vec { - let span = ecx.with_def_site_ctxt(span); + let eii_attr_span = ecx.with_def_site_ctxt(eii_attr_span); - let (item, stmt) = if let Annotatable::Item(item) = item { - (item, false) + let (item, wrap_item): (_, &dyn Fn(_) -> _) = if let Annotatable::Item(item) = item { + (item, &Annotatable::Item) } else if let Annotatable::Stmt(ref stmt) = item && let StmtKind::Item(ref item) = stmt.kind { - (item.clone(), true) + (item.clone(), &|item| { + Annotatable::Stmt(Box::new(Stmt { + id: DUMMY_NODE_ID, + kind: StmtKind::Item(item), + span: eii_attr_span, + })) + }) } else { ecx.dcx().emit_err(EiiSharedMacroExpectedFunction { - span, + span: eii_attr_span, name: path_to_string(&meta_item.path), }); return vec![item]; }; - let orig_item = item.clone(); - - let item = *item; - - let ast::Item { attrs, id: _, span: item_span, vis, kind: ItemKind::Fn(mut func), tokens: _ } = - item + let ast::Item { attrs, id: _, span: _, vis, kind: ItemKind::Fn(func), tokens: _ } = + item.as_ref() else { ecx.dcx().emit_err(EiiSharedMacroExpectedFunction { - span, + span: eii_attr_span, name: path_to_string(&meta_item.path), }); - return vec![Annotatable::Item(Box::new(item))]; + return vec![wrap_item(item)]; + }; + // only clone what we need + let attrs = attrs.clone(); + let func = (**func).clone(); + let vis = vis.clone(); + + let attrs_from_decl = + filter_attrs_for_multiple_eii_attr(ecx, attrs, eii_attr_span, &meta_item.path); + + let Ok(macro_name) = name_for_impl_macro(ecx, &func, &meta_item) else { + return vec![wrap_item(item)]; }; - // Detect when this is the *second* eii attribute on an item. - let mut new_attrs = ThinVec::new(); - for i in attrs { - if i.has_name(sym::eii) { - ecx.dcx().emit_err(EiiOnlyOnce { - span: i.span, - first_span: span, - name: path_to_string(&meta_item.path), - }); - } else { - new_attrs.push(i); - } - } - let attrs = new_attrs; - - let macro_name = if meta_item.is_word() { - func.ident - } else if let Some([first]) = meta_item.meta_item_list() - && let Some(m) = first.meta_item() - && m.path.segments.len() == 1 - { - m.path.segments[0].ident - } else { - ecx.dcx().emit_err(EiiMacroExpectedMaxOneArgument { - span: meta_item.span, - name: path_to_string(&meta_item.path), - }); - return vec![Annotatable::Item(orig_item)]; - }; + // span of the declaring item without attributes + let item_span = func.sig.span; + // span of the eii attribute and the item below it, i.e. the full declaration + let decl_span = eii_attr_span.to(item_span); + let foreign_item_name = func.ident; let mut return_items = Vec::new(); if func.body.is_some() { - let mut default_func = func.clone(); - func.body = None; - default_func.eii_impls.push(ast::EiiImpl { - node_id: DUMMY_NODE_ID, - eii_macro_path: ast::Path::from_ident(macro_name), - impl_safety: if impl_unsafe { ast::Safety::Unsafe(span) } else { ast::Safety::Default }, - span, - inner_span: macro_name.span, - is_default: true, // important! - }); - - return_items.push(Box::new(ast::Item { - attrs: ThinVec::new(), - id: ast::DUMMY_NODE_ID, - span, - vis: ast::Visibility { span, kind: ast::VisibilityKind::Inherited, tokens: None }, - kind: ast::ItemKind::Const(Box::new(ast::ConstItem { - ident: Ident { name: kw::Underscore, span }, - defaultness: ast::Defaultness::Final, - generics: ast::Generics::default(), - ty: Box::new(ast::Ty { - id: DUMMY_NODE_ID, - kind: ast::TyKind::Tup(ThinVec::new()), - span, - tokens: None, - }), - rhs: Some(ast::ConstItemRhs::Body(Box::new(ast::Expr { - id: DUMMY_NODE_ID, - kind: ast::ExprKind::Block( - Box::new(ast::Block { - stmts: thin_vec![ast::Stmt { - id: DUMMY_NODE_ID, - kind: ast::StmtKind::Item(Box::new(ast::Item { - attrs: thin_vec![], // FIXME: re-add some original attrs - id: DUMMY_NODE_ID, - span: item_span, - vis: ast::Visibility { - span, - kind: ast::VisibilityKind::Inherited, - tokens: None - }, - kind: ItemKind::Fn(default_func), - tokens: None, - })), - span - }], - id: DUMMY_NODE_ID, - rules: ast::BlockCheckMode::Default, - span, - tokens: None, - }), - None, - ), - span, - attrs: ThinVec::new(), - tokens: None, - }))), - define_opaque: None, - })), - tokens: None, - })) + return_items.push(Box::new(generate_default_impl( + &func, + impl_unsafe, + macro_name, + eii_attr_span, + item_span, + ))) } - let decl_span = span.to(func.sig.span); + return_items.push(Box::new(generate_foreign_item( + ecx, + eii_attr_span, + item_span, + func, + vis, + &attrs_from_decl, + ))); + return_items.push(Box::new(generate_attribute_macro_to_implement( + ecx, + eii_attr_span, + macro_name, + foreign_item_name, + impl_unsafe, + decl_span, + ))); + + return_items.into_iter().map(wrap_item).collect() +} + +/// Decide on the name of the macro that can be used to implement the EII. +/// This is either an explicitly given name, or the name of the item in the +/// declaration of the EII. +fn name_for_impl_macro( + ecx: &mut ExtCtxt<'_>, + func: &ast::Fn, + meta_item: &MetaItem, +) -> Result { + if meta_item.is_word() { + Ok(func.ident) + } else if let Some([first]) = meta_item.meta_item_list() + && let Some(m) = first.meta_item() + && m.path.segments.len() == 1 + { + Ok(m.path.segments[0].ident) + } else { + Err(ecx.dcx().emit_err(EiiMacroExpectedMaxOneArgument { + span: meta_item.span, + name: path_to_string(&meta_item.path), + })) + } +} + +/// Ensure that in the list of attrs, there's only a single `eii` attribute. +fn filter_attrs_for_multiple_eii_attr( + ecx: &mut ExtCtxt<'_>, + attrs: ThinVec, + eii_attr_span: Span, + eii_attr_path: &Path, +) -> ThinVec { + attrs + .into_iter() + .filter(|i| { + if i.has_name(sym::eii) { + ecx.dcx().emit_err(EiiOnlyOnce { + span: i.span, + first_span: eii_attr_span, + name: path_to_string(eii_attr_path), + }); + false + } else { + true + } + }) + .collect() +} + +fn generate_default_impl( + func: &ast::Fn, + impl_unsafe: bool, + macro_name: Ident, + eii_attr_span: Span, + item_span: Span, +) -> ast::Item { + // FIXME: re-add some original attrs + let attrs = ThinVec::new(); + + let mut default_func = func.clone(); + default_func.eii_impls.push(EiiImpl { + node_id: DUMMY_NODE_ID, + inner_span: macro_name.span, + eii_macro_path: ast::Path::from_ident(macro_name), + impl_safety: if impl_unsafe { + ast::Safety::Unsafe(eii_attr_span) + } else { + ast::Safety::Default + }, + span: eii_attr_span, + is_default: true, + }); + + ast::Item { + attrs: ThinVec::new(), + id: ast::DUMMY_NODE_ID, + span: eii_attr_span, + vis: ast::Visibility { + span: eii_attr_span, + kind: ast::VisibilityKind::Inherited, + tokens: None, + }, + kind: ast::ItemKind::Const(Box::new(ast::ConstItem { + ident: Ident { name: kw::Underscore, span: eii_attr_span }, + defaultness: ast::Defaultness::Final, + generics: ast::Generics::default(), + ty: Box::new(ast::Ty { + id: DUMMY_NODE_ID, + kind: ast::TyKind::Tup(ThinVec::new()), + span: eii_attr_span, + tokens: None, + }), + rhs: Some(ast::ConstItemRhs::Body(Box::new(ast::Expr { + id: DUMMY_NODE_ID, + kind: ast::ExprKind::Block( + Box::new(ast::Block { + stmts: thin_vec![ast::Stmt { + id: DUMMY_NODE_ID, + kind: ast::StmtKind::Item(Box::new(ast::Item { + attrs, + id: DUMMY_NODE_ID, + span: item_span, + vis: ast::Visibility { + span: eii_attr_span, + kind: ast::VisibilityKind::Inherited, + tokens: None + }, + kind: ItemKind::Fn(Box::new(default_func)), + tokens: None, + })), + span: eii_attr_span + }], + id: DUMMY_NODE_ID, + rules: ast::BlockCheckMode::Default, + span: eii_attr_span, + tokens: None, + }), + None, + ), + span: eii_attr_span, + attrs: ThinVec::new(), + tokens: None, + }))), + define_opaque: None, + })), + tokens: None, + } +} + +/// Generates a foreign item, like +/// +/// ```rust, ignore +/// extern "…" { safe fn item(); } +/// ``` +fn generate_foreign_item( + ecx: &mut ExtCtxt<'_>, + eii_attr_span: Span, + item_span: Span, + mut func: ast::Fn, + vis: Visibility, + attrs_from_decl: &[Attribute], +) -> ast::Item { + let mut foreign_item_attrs = ThinVec::new(); + foreign_item_attrs.extend_from_slice(attrs_from_decl); + + // Add the rustc_eii_extern_item on the foreign item. Usually, foreign items are mangled. + // This attribute makes sure that we later know that this foreign item's symbol should not be. + foreign_item_attrs.push(ecx.attr_word(sym::rustc_eii_extern_item, eii_attr_span)); let abi = match func.sig.header.ext { // extern "X" fn => extern "X" {} @@ -196,85 +297,69 @@ fn eii_( suffix: None, symbol_unescaped: sym::Rust, style: ast::StrStyle::Cooked, - span, + span: eii_attr_span, }), }; // ABI has been moved to the extern {} block, so we remove it from the fn item. func.sig.header.ext = ast::Extern::None; + func.body = None; // And mark safe functions explicitly as `safe fn`. if func.sig.header.safety == ast::Safety::Default { func.sig.header.safety = ast::Safety::Safe(func.sig.span); } - // extern "…" { safe fn item(); } - let mut extern_item_attrs = attrs.clone(); - extern_item_attrs.push(ast::Attribute { - kind: ast::AttrKind::Normal(Box::new(ast::NormalAttr { - item: ast::AttrItem { - unsafety: ast::Safety::Default, - // Add the rustc_eii_extern_item on the foreign item. Usually, foreign items are mangled. - // This attribute makes sure that we later know that this foreign item's symbol should not be. - path: ast::Path::from_ident(Ident::new(sym::rustc_eii_extern_item, span)), - args: ast::AttrArgs::Empty, - tokens: None, - }, - tokens: None, - })), - id: ecx.sess.psess.attr_id_generator.mk_attr_id(), - style: ast::AttrStyle::Outer, - span, - }); - - let extern_block = Box::new(ast::Item { + ast::Item { attrs: ast::AttrVec::default(), id: ast::DUMMY_NODE_ID, - span, - vis: ast::Visibility { span, kind: ast::VisibilityKind::Inherited, tokens: None }, + span: eii_attr_span, + vis: ast::Visibility { + span: eii_attr_span, + kind: ast::VisibilityKind::Inherited, + tokens: None, + }, kind: ast::ItemKind::ForeignMod(ast::ForeignMod { - extern_span: span, - safety: ast::Safety::Unsafe(span), + extern_span: eii_attr_span, + safety: ast::Safety::Unsafe(eii_attr_span), abi, items: From::from([Box::new(ast::ForeignItem { - attrs: extern_item_attrs, + attrs: foreign_item_attrs, id: ast::DUMMY_NODE_ID, span: item_span, vis, - kind: ast::ForeignItemKind::Fn(func.clone()), + kind: ast::ForeignItemKind::Fn(Box::new(func.clone())), tokens: None, })]), }), tokens: None, - }); + } +} - let mut macro_attrs = attrs.clone(); - macro_attrs.push( - // #[builtin_macro(eii_shared_macro)] - ast::Attribute { - kind: ast::AttrKind::Normal(Box::new(ast::NormalAttr { - item: ast::AttrItem { - unsafety: ast::Safety::Default, - path: ast::Path::from_ident(Ident::new(sym::rustc_builtin_macro, span)), - args: ast::AttrArgs::Delimited(ast::DelimArgs { - dspan: DelimSpan::from_single(span), - delim: Delimiter::Parenthesis, - tokens: TokenStream::new(vec![tokenstream::TokenTree::token_alone( - token::TokenKind::Ident(sym::eii_shared_macro, token::IdentIsRaw::No), - span, - )]), - }), - tokens: None, - }, - tokens: None, - })), - id: ecx.sess.psess.attr_id_generator.mk_attr_id(), - style: ast::AttrStyle::Outer, - span, - }, - ); +/// Generate a stub macro (a bit like in core) that will roughly look like: +/// +/// ```rust, ignore, example +/// // Since this a stub macro, the actual code that expands it lives in the compiler. +/// // This attribute tells the compiler that +/// #[builtin_macro(eii_shared_macro)] +/// // the metadata to link this macro to the generated foreign item. +/// #[eii_extern_target()] +/// macro macro_name { () => {} } +/// ``` +fn generate_attribute_macro_to_implement( + ecx: &mut ExtCtxt<'_>, + span: Span, + macro_name: Ident, + foreign_item_name: Ident, + impl_unsafe: bool, + decl_span: Span, +) -> ast::Item { + let mut macro_attrs = ThinVec::new(); - let macro_def = Box::new(ast::Item { + // #[builtin_macro(eii_shared_macro)] + macro_attrs.push(ecx.attr_nested_word(sym::rustc_builtin_macro, sym::eii_shared_macro, span)); + + ast::Item { attrs: macro_attrs, id: ast::DUMMY_NODE_ID, span, @@ -305,33 +390,15 @@ fn eii_( ]), }), macro_rules: false, - // #[eii_extern_target(func.ident)] + // #[eii_extern_target(foreign_item_ident)] eii_extern_target: Some(ast::EiiExternTarget { - extern_item_path: ast::Path::from_ident(func.ident), + extern_item_path: ast::Path::from_ident(foreign_item_name), impl_unsafe, span: decl_span, }), }, ), tokens: None, - }); - - return_items.push(extern_block); - return_items.push(macro_def); - - if stmt { - return_items - .into_iter() - .map(|i| { - Annotatable::Stmt(Box::new(Stmt { - id: DUMMY_NODE_ID, - kind: StmtKind::Item(i), - span, - })) - }) - .collect() - } else { - return_items.into_iter().map(|i| Annotatable::Item(i)).collect() } } @@ -436,10 +503,10 @@ pub(crate) fn eii_shared_macro( f.eii_impls.push(EiiImpl { node_id: DUMMY_NODE_ID, + inner_span: meta_item.path.span, eii_macro_path: meta_item.path.clone(), impl_safety: meta_item.unsafety, span, - inner_span: meta_item.path.span, is_default, }); diff --git a/compiler/rustc_builtin_macros/src/errors.rs b/compiler/rustc_builtin_macros/src/errors.rs index 2a4c499349ad..ad31eae10f60 100644 --- a/compiler/rustc_builtin_macros/src/errors.rs +++ b/compiler/rustc_builtin_macros/src/errors.rs @@ -216,17 +216,6 @@ mod autodiff { } } -pub(crate) use ad_fallback::*; -mod ad_fallback { - use super::*; - #[derive(Diagnostic)] - #[diag(builtin_macros_autodiff_not_build)] - pub(crate) struct AutoDiffSupportNotBuild { - #[primary_span] - pub(crate) span: Span, - } -} - #[derive(Diagnostic)] #[diag(builtin_macros_concat_bytes_invalid)] pub(crate) struct ConcatBytesInvalid { diff --git a/compiler/rustc_builtin_macros/src/source_util.rs b/compiler/rustc_builtin_macros/src/source_util.rs index d49f4ded010b..0f31de60a8a4 100644 --- a/compiler/rustc_builtin_macros/src/source_util.rs +++ b/compiler/rustc_builtin_macros/src/source_util.rs @@ -144,9 +144,7 @@ pub(crate) fn expand_include<'cx>( let mut p = unwrap_or_emit_fatal(new_parser_from_file( self.psess, &self.path, - // Don't strip frontmatter for backward compatibility, `---` may be the start of a - // manifold negation. FIXME: Ideally, we wouldn't strip shebangs here either. - StripTokens::Shebang, + StripTokens::Nothing, Some(self.span), )); let expr = parse_expr(&mut p).ok()?; diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/audit.yml b/compiler/rustc_codegen_cranelift/.github/workflows/audit.yml index 27c95572ef87..274b9504beb0 100644 --- a/compiler/rustc_codegen_cranelift/.github/workflows/audit.yml +++ b/compiler/rustc_codegen_cranelift/.github/workflows/audit.yml @@ -12,7 +12,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: | - sed -i 's/components.*/components = []/' rust-toolchain + sed -i 's/components.*/components = []/' rust-toolchain.toml - uses: rustsec/audit-check@v1.4.1 with: token: ${{ secrets.GITHUB_TOKEN }} diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml index 0930b924d177..07d9af4a9b54 100644 --- a/compiler/rustc_codegen_cranelift/.github/workflows/main.yml +++ b/compiler/rustc_codegen_cranelift/.github/workflows/main.yml @@ -28,7 +28,7 @@ jobs: - name: Avoid installing rustc-dev run: | - sed -i 's/components.*/components = ["rustfmt"]/' rust-toolchain + sed -i 's/components.*/components = ["rustfmt"]/' rust-toolchain.toml rustfmt -v - name: Rustfmt @@ -88,7 +88,7 @@ jobs: uses: actions/cache@v4 with: path: build/cg_clif - key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} + key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-cargo-build-target-${{ hashFiles('rust-toolchain.toml', '**/Cargo.lock') }} - name: Set MinGW as the default toolchain if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu' @@ -158,7 +158,7 @@ jobs: uses: actions/cache@v4 with: path: build/cg_clif - key: ${{ runner.os }}-x86_64-unknown-linux-gnu-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} + key: ${{ runner.os }}-x86_64-unknown-linux-gnu-cargo-build-target-${{ hashFiles('rust-toolchain.toml', '**/Cargo.lock') }} - name: Install hyperfine run: | @@ -207,7 +207,7 @@ jobs: uses: actions/cache@v4 with: path: build/cg_clif - key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-dist-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} + key: ${{ runner.os }}-${{ matrix.env.TARGET_TRIPLE }}-dist-cargo-build-target-${{ hashFiles('rust-toolchain.toml', '**/Cargo.lock') }} - name: Set MinGW as the default toolchain if: matrix.os == 'windows-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu' diff --git a/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml b/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml index 9253ab96353c..b22725fdc9d4 100644 --- a/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml +++ b/compiler/rustc_codegen_cranelift/.github/workflows/rustc.yml @@ -20,7 +20,7 @@ jobs: uses: actions/cache@v4 with: path: build/cg_clif - key: ${{ runner.os }}-rustc-test-cargo-build-target-${{ hashFiles('rust-toolchain', 'Cargo.lock') }} + key: ${{ runner.os }}-rustc-test-cargo-build-target-${{ hashFiles('rust-toolchain.toml', 'Cargo.lock') }} - name: Test run: ./scripts/test_bootstrap.sh @@ -40,7 +40,7 @@ jobs: uses: actions/cache@v4 with: path: build/cg_clif - key: ${{ runner.os }}-rustc-test-cargo-build-target-${{ hashFiles('rust-toolchain', 'Cargo.lock') }} + key: ${{ runner.os }}-rustc-test-cargo-build-target-${{ hashFiles('rust-toolchain.toml', 'Cargo.lock') }} - name: Install ripgrep run: | diff --git a/compiler/rustc_codegen_cranelift/Cargo.lock b/compiler/rustc_codegen_cranelift/Cargo.lock index 617c7f0e34cd..3d13b5540e19 100644 --- a/compiler/rustc_codegen_cranelift/Cargo.lock +++ b/compiler/rustc_codegen_cranelift/Cargo.lock @@ -43,42 +43,42 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cranelift-assembler-x64" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7631e609c97f063f9777aae405e8492abf9bf92336d7aa3f875403dd4ffd7d" +checksum = "8bd963a645179fa33834ba61fa63353998543b07f877e208da9eb47d4a70d1e7" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c030edccdc4a5bbf28fbfe7701b5cd1f9854b4445184dd34af2a7e8f8db6f45" +checksum = "3f6d5739c9dc6b5553ca758d78d87d127dd19f397f776efecf817b8ba8d0bb01" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb544c1242d0ca98baf01873ebba96c79d5df155d5108d9bb699aefc741f5e6d" +checksum = "ff402c11bb1c9652b67a3e885e84b1b8d00c13472c8fd85211e06a41a63c3e03" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0325aecbafec053d3d3f082edfdca7937e2945e7f09c5ff9672e05198312282" +checksum = "769a0d88c2f5539e9c5536a93a7bf164b0dc68d91e3d00723e5b4ffc1440afdc" [[package]] name = "cranelift-codegen" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb3236fd319ae897ba00c8a25105081de5c1348576def0e96c062ad259f87a7" +checksum = "d4351f721fb3b26add1c180f0a75c7474bab2f903c8b777c6ca65238ded59a78" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -102,9 +102,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b8791c911a361c539130ace34fb726b16aca4216470ec75d75264b1495c8a3a" +checksum = "61f86c0ba5b96713643f4dd0de0df12844de9c7bb137d6829b174b706939aa74" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -114,33 +114,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ead718c2a10990870c19b2497b5a04b8aae6024485e33da25b5d02e35819e0" +checksum = "f08605eee8d51fd976a970bd5b16c9529b51b624f8af68f80649ffb172eb85a4" [[package]] name = "cranelift-control" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0a57fc972b5651047efddccb99440d103d9d8c13393ccebde15ddd5b6a1181b" +checksum = "623aab0a09e40f0cf0b5d35eb7832bae4c4f13e3768228e051a6c1a60e88ef5f" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aae980b4a1678b601eab2f52e372ed0b3c9565a31c17f380008cb97b3a699c5" +checksum = "ea0f066e07e3bcbe38884cc5c94c32c7a90267d69df80f187d9dfe421adaa7c4" dependencies = [ "cranelift-bitset", ] [[package]] name = "cranelift-frontend" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a78877016b607982ca1708c0dd4ce23bde04581a39854c9b43a1dca43625b54c" +checksum = "40865b02a0e52ca8e580ad64feef530cb1d05f6bb4972b4eef05e3eaeae81701" dependencies = [ "cranelift-codegen", "log", @@ -150,15 +150,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dc46a68b46d4f53f9f2f02ab8d3a34b00f03a21c124a7a965b8cbf5fdb6773b" +checksum = "104b3c117ae513e9af1d90679842101193a5ccb96ac9f997966d85ea25be2852" [[package]] name = "cranelift-jit" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df920009af919ad9df52eb7b47b1895145822e0c29da9b715a876fc8ecc6d82" +checksum = "3aa5f855cfb8e4253ed2d0dfc1a0b6ebe4912e67aa8b7ee14026ff55ca17f1fe" dependencies = [ "anyhow", "cranelift-codegen", @@ -171,14 +171,14 @@ dependencies = [ "region", "target-lexicon", "wasmtime-internal-jit-icache-coherence", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "cranelift-module" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddcf313629071ce74de8e59f02092f5453d1a01047607fc4ad36886b8bd1486c" +checksum = "b1d01806b191b59f4fc4680293dd5f554caf2de5b62f95eff5beef7acb46c29c" dependencies = [ "anyhow", "cranelift-codegen", @@ -187,9 +187,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03faa07ec8cf373250a8252eb773d098ff88259fa1c19ee1ecde8012839f4097" +checksum = "e5c54e0a358bc05b48f2032e1c320e7f468da068604f2869b77052eab68eb0fe" dependencies = [ "cranelift-codegen", "libc", @@ -198,9 +198,9 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cca62c14f3c2e4f438192562bbf82d1a98a59543cc66ba04fb658ba99f515a6" +checksum = "3d17e0216be5daabab616647c1918e06dae0708474ba5f7b7762ac24ea5eb126" dependencies = [ "anyhow", "cranelift-codegen", @@ -213,9 +213,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.126.0" +version = "0.127.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0484cb32c527a742e1bba09ef174acac0afb1dcf623ef1adda42849200edcd2e" +checksum = "cc6f4b039f453b66c75e9f7886e5a2af96276e151f44dc19b24b58f9a0c98009" [[package]] name = "crc32fast" @@ -293,7 +293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" dependencies = [ "cfg-if", - "windows-link 0.2.1", + "windows-link", ] [[package]] @@ -469,31 +469,25 @@ checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f67986f5c499274ae5b2ba5b173bba0b95d1381f5ca70d8eec657f2392117d8" +checksum = "0858b470463f3e7c73acd6049046049e64be17b98901c2db5047450cf83df1fe" dependencies = [ "anyhow", "cfg-if", "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "wasmtime-internal-math" -version = "39.0.0" +version = "40.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a681733e9b5d5d8804ee6cacd59f92c0d87ba2274f42ee1d4e5a943828d0075d" +checksum = "222e1a590ece4e898f20af1e541b61d2cb803f2557e7eaff23e6c1db5434454a" dependencies = [ "libm", ] -[[package]] -name = "windows-link" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" - [[package]] name = "windows-link" version = "0.2.1" @@ -506,16 +500,16 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.6", + "windows-targets", ] [[package]] name = "windows-sys" -version = "0.60.2" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-targets 0.53.3", + "windows-link", ] [[package]] @@ -524,31 +518,14 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows-targets" -version = "0.53.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" -dependencies = [ - "windows-link 0.1.3", - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] @@ -557,92 +534,44 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" - [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" - [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" - [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" - [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" - [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" - [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" diff --git a/compiler/rustc_codegen_cranelift/Cargo.toml b/compiler/rustc_codegen_cranelift/Cargo.toml index 58e61cd0b9d7..ee4bde477c47 100644 --- a/compiler/rustc_codegen_cranelift/Cargo.toml +++ b/compiler/rustc_codegen_cranelift/Cargo.toml @@ -8,12 +8,12 @@ crate-type = ["dylib"] [dependencies] # These have to be in sync with each other -cranelift-codegen = { version = "0.126.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] } -cranelift-frontend = { version = "0.126.0" } -cranelift-module = { version = "0.126.0" } -cranelift-native = { version = "0.126.0" } -cranelift-jit = { version = "0.126.0", optional = true } -cranelift-object = { version = "0.126.0" } +cranelift-codegen = { version = "0.127.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] } +cranelift-frontend = { version = "0.127.0" } +cranelift-module = { version = "0.127.0" } +cranelift-native = { version = "0.127.0" } +cranelift-jit = { version = "0.127.0", optional = true } +cranelift-object = { version = "0.127.0" } target-lexicon = "0.13" gimli = { version = "0.32", default-features = false, features = ["write"] } object = { version = "0.37.3", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } @@ -24,12 +24,12 @@ smallvec = "1.8.1" [patch.crates-io] # Uncomment to use an unreleased version of cranelift -#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-39.0.0" } -#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-39.0.0" } -#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-39.0.0" } -#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-39.0.0" } -#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-39.0.0" } -#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-39.0.0" } +#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-40.0.0" } +#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-40.0.0" } +#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-40.0.0" } +#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-40.0.0" } +#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-40.0.0" } +#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-40.0.0" } # Uncomment to use local checkout of cranelift #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } diff --git a/compiler/rustc_codegen_cranelift/example/std_example.rs b/compiler/rustc_codegen_cranelift/example/std_example.rs index c569ef0ef829..33db75f0943a 100644 --- a/compiler/rustc_codegen_cranelift/example/std_example.rs +++ b/compiler/rustc_codegen_cranelift/example/std_example.rs @@ -259,6 +259,9 @@ unsafe fn test_simd() { test_mm_cvttps_epi32(); test_mm_cvtsi128_si64(); + #[cfg(not(jit))] + test_mm_cvtps_ph(); + test_mm_extract_epi8(); test_mm_insert_epi16(); test_mm_shuffle_epi8(); @@ -558,6 +561,21 @@ unsafe fn test_mm_cvttps_epi32() { } } +#[cfg(target_arch = "x86_64")] +#[target_feature(enable = "f16c")] +#[cfg(not(jit))] +unsafe fn test_mm_cvtps_ph() { + const F16_ONE: i16 = 0x3c00; + const F16_TWO: i16 = 0x4000; + const F16_THREE: i16 = 0x4200; + const F16_FOUR: i16 = 0x4400; + + let a = _mm_set_ps(1.0, 2.0, 3.0, 4.0); + let r = _mm_cvtps_ph::<_MM_FROUND_CUR_DIRECTION>(a); + let e = _mm_set_epi16(0, 0, 0, 0, F16_ONE, F16_TWO, F16_THREE, F16_FOUR); + assert_eq_m128i(r, e); +} + fn test_checked_mul() { let u: Option = u8::from_str_radix("1000", 10).ok(); assert_eq!(u, None); diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain deleted file mode 100644 index 461dbcdb0fb5..000000000000 --- a/compiler/rustc_codegen_cranelift/rust-toolchain +++ /dev/null @@ -1,4 +0,0 @@ -[toolchain] -channel = "nightly-2025-12-08" -components = ["rust-src", "rustc-dev", "llvm-tools"] -profile = "minimal" diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain.toml b/compiler/rustc_codegen_cranelift/rust-toolchain.toml new file mode 100644 index 000000000000..b157c5879ba7 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "nightly-2025-12-23" +components = ["rust-src", "rustc-dev", "llvm-tools", "rustfmt"] +profile = "minimal" diff --git a/compiler/rustc_codegen_cranelift/scripts/rustup.sh b/compiler/rustc_codegen_cranelift/scripts/rustup.sh index 1a82193303e2..e26be307328f 100755 --- a/compiler/rustc_codegen_cranelift/scripts/rustup.sh +++ b/compiler/rustc_codegen_cranelift/scripts/rustup.sh @@ -22,8 +22,7 @@ case $1 in "prepare") echo "=> Installing new nightly" rustup toolchain install --profile minimal "nightly-${TOOLCHAIN}" # Sanity check to see if the nightly exists - sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain - rustup component add rustfmt || true + sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain.toml echo "=> Uninstalling all old nightlies" for nightly in $(rustup toolchain list | grep nightly | grep -v "$TOOLCHAIN" | grep -v nightly-x86_64); do @@ -35,7 +34,7 @@ case $1 in ./y.sh prepare ;; "commit") - git add rust-toolchain + git add rust-toolchain.toml git commit -m "Rustup to $(rustc -V)" ;; "push") diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh index b5af585a732e..b25269d1430a 100755 --- a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh +++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh @@ -35,6 +35,7 @@ git checkout -- tests/ui/entry-point/auxiliary/bad_main_functions.rs rm tests/ui/asm/x86_64/evex512-implicit-feature.rs # unimplemented AVX512 x86 vendor intrinsic rm tests/ui/simd/dont-invalid-bitcast-x86_64.rs # unimplemented llvm.x86.sse41.round.ps rm tests/ui/simd/intrinsic/generic-arithmetic-pass.rs # unimplemented simd_funnel_{shl,shr} +rm -r tests/ui/scalable-vectors # scalable vectors are unsupported # exotic linkages rm tests/incremental/hashes/function_interfaces.rs @@ -53,23 +54,29 @@ rm tests/ui/sanitizer/kcfi-c-variadic.rs # same rm tests/ui/c-variadic/same-program-multiple-abis-x86_64.rs # variadics for calling conventions other than C unsupported rm tests/ui/delegation/fn-header.rs +# inline assembly features +rm tests/ui/asm/x86_64/issue-96797.rs # const and sym inline asm operands don't work entirely correctly +rm tests/ui/asm/global-asm-mono-sym-fn.rs # same +rm tests/ui/asm/naked-asm-mono-sym-fn.rs # same +rm tests/ui/asm/x86_64/goto.rs # inline asm labels not supported +rm tests/ui/asm/label-operand.rs # same +rm tests/ui/asm/may_unwind.rs # asm unwinding not supported +rm tests/ui/asm/aarch64/may_unwind.rs # same + # misc unimplemented things rm tests/ui/target-feature/missing-plusminus.rs # error not implemented rm -r tests/run-make/repr128-dwarf # debuginfo test rm -r tests/run-make/split-debuginfo # same rm -r tests/run-make/target-specs # i686 not supported by Cranelift rm -r tests/run-make/mismatching-target-triples # same -rm tests/ui/asm/x86_64/issue-96797.rs # const and sym inline asm operands don't work entirely correctly -rm tests/ui/asm/global-asm-mono-sym-fn.rs # same -rm tests/ui/asm/naked-asm-mono-sym-fn.rs # same -rm tests/ui/asm/x86_64/goto.rs # inline asm labels not supported -rm tests/ui/asm/label-operand.rs # same rm tests/ui/simd/simd-bitmask-notpow2.rs # non-pow-of-2 simd vector sizes rm -r tests/run-make/used-proc-macro # used(linker) isn't supported yet rm tests/ui/linking/no-gc-encapsulation-symbols.rs # same rm tests/ui/attributes/fn-align-dyn.rs # per-function alignment not supported rm -r tests/ui/explicit-tail-calls # tail calls rm -r tests/run-make/pointer-auth-link-with-c # pointer auth +rm -r tests/ui/eii # EII not yet implemented +rm -r tests/run-make/forced-unwind-terminate-pof # forced unwinding doesn't take precedence # requires LTO rm -r tests/run-make/cdylib @@ -78,6 +85,7 @@ rm -r tests/run-make/lto-* rm -r tests/run-make/reproducible-build-2 rm -r tests/run-make/no-builtins-lto rm -r tests/run-make/reachable-extern-fn-available-lto +rm -r tests/run-make/no-builtins-linker-plugin-lto # coverage instrumentation rm tests/ui/consts/precise-drop-with-coverage.rs @@ -87,6 +95,7 @@ rm -r tests/ui/instrument-coverage/ # ================== rm tests/ui/codegen/issue-28950.rs # depends on stack size optimizations rm tests/ui/codegen/init-large-type.rs # same +rm tests/ui/codegen/StackColoring-not-blowup-stack-issue-40883.rs # same rm tests/ui/statics/const_generics.rs # tests an optimization rm tests/ui/linking/executable-no-mangle-strip.rs # requires --gc-sections to work for statics @@ -143,6 +152,15 @@ rm tests/ui/errors/remap-path-prefix-sysroot.rs # different sysroot source path rm -r tests/run-make/export/extern-opt # something about rustc version mismatches rm -r tests/run-make/export # same rm -r tests/ui/compiletest-self-test/compile-flags-incremental.rs # needs compiletest compiled with panic=unwind +rm tests/ui/async-await/in-trait/dont-project-to-specializable-projection.rs # something going wrong with stdlib source remapping +rm tests/ui/consts/miri_unleashed/drop.rs # same +rm tests/ui/error-emitter/multiline-removal-suggestion.rs # same +rm tests/ui/lint/lint-const-item-mutation.rs # same +rm tests/ui/lint/use-redundant/use-redundant-issue-71450.rs # same +rm tests/ui/lint/use-redundant/use-redundant-prelude-rust-2021.rs # same +rm tests/ui/specialization/const_trait_impl.rs # same +rm tests/ui/thir-print/offset_of.rs # same +rm tests/ui/traits/const-traits/const_closure-const_trait_impl-ice-113381.rs # same # genuine bugs # ============ @@ -157,6 +175,7 @@ rm tests/ui/lint/non-snake-case/lint-non-snake-case-crate.rs # same rm tests/ui/async-await/async-drop/async-drop-initial.rs # same (rust-lang/rust#140493) rm -r tests/ui/codegen/equal-pointers-unequal # make incorrect assumptions about the location of stack variables rm -r tests/run-make-cargo/rustdoc-scrape-examples-paths # FIXME(rust-lang/rust#145580) incr comp bug +rm -r tests/incremental/extern_static/issue-49153.rs # assumes reference to undefined static gets optimized away rm tests/ui/intrinsics/panic-uninitialized-zeroed.rs # really slow with unoptimized libstd rm tests/ui/process/process-panic-after-fork.rs # same diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index ac5b3c240785..1a916c876824 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -10,7 +10,7 @@ use rustc_data_structures::profiling::SelfProfilerRef; use rustc_index::IndexVec; use rustc_middle::ty::TypeVisitableExt; use rustc_middle::ty::adjustment::PointerCoercion; -use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv}; +use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::print::with_no_trimmed_paths; use rustc_session::config::OutputFilenames; use rustc_span::Symbol; @@ -853,17 +853,6 @@ fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: fx.bcx.ins().nop(); } } - Rvalue::NullaryOp(ref null_op) => { - assert!(lval.layout().ty.is_sized(fx.tcx, fx.typing_env())); - let val = match null_op { - NullOp::RuntimeChecks(kind) => kind.value(fx.tcx.sess), - }; - let val = CValue::by_val( - fx.bcx.ins().iconst(types::I8, i64::from(val)), - fx.layout_of(fx.tcx.types.bool), - ); - lval.write_cvalue(fx, val); - } Rvalue::Aggregate(ref kind, ref operands) if matches!(**kind, AggregateKind::RawPtr(..)) => { @@ -1050,6 +1039,11 @@ pub(crate) fn codegen_operand<'tcx>( cplace.to_cvalue(fx) } Operand::Constant(const_) => crate::constant::codegen_constant_operand(fx, const_), + Operand::RuntimeChecks(checks) => { + let val = checks.value(fx.tcx.sess); + let layout = fx.layout_of(fx.tcx.types.bool); + return CValue::const_val(fx, layout, val.into()); + } } } diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index 2b65b8290681..ff8e6744bd32 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -540,6 +540,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( operand: &Operand<'tcx>, ) -> Option { match operand { + Operand::RuntimeChecks(checks) => Some(checks.value(fx.tcx.sess).into()), Operand::Constant(const_) => eval_mir_constant(fx, const_).0.try_to_scalar_int(), // FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored // inside a temporary before being passed to the intrinsic requiring the const argument. diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index 9dba46363936..3a8ca25a5fc0 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -190,7 +190,7 @@ fn dep_symbol_lookup_fn( diag.emit(); } Linkage::Dynamic => { - dylib_paths.push(src.dylib.as_ref().unwrap().0.clone()); + dylib_paths.push(src.dylib.as_ref().unwrap().clone()); } } } diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs index 37fbe4be1b0f..61f48fa97743 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs @@ -1313,6 +1313,35 @@ pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>( ret.write_cvalue_transmute(fx, res); } + "llvm.x86.vcvtps2ph.128" => { + // https://www.intel.com/content/www/us/en/docs/intrinsics-guide/index.html#text=_mm_cvtps_ph + intrinsic_args!(fx, args => (a, _imm8); intrinsic); + let a = a.load_scalar(fx); + + let imm8 = + if let Some(imm8) = crate::constant::mir_operand_get_const_val(fx, &args[1].node) { + imm8 + } else { + fx.tcx + .dcx() + .span_fatal(span, "Index argument for `_mm_cvtps_ph` is not a constant"); + }; + + let imm8 = imm8.to_u32(); + + codegen_inline_asm_inner( + fx, + &[InlineAsmTemplatePiece::String(format!("vcvtps2ph xmm0, xmm0, {imm8}").into())], + &[CInlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::xmm0)), + _late: true, + in_value: a, + out_place: Some(ret), + }], + InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM, + ); + } + _ => { fx.tcx .dcx() diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index 0bce31beb8b8..bef9c6747457 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -130,7 +130,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( return; } - let idx = generic_args[2].expect_const().to_value().valtree.unwrap_branch(); + let idx = generic_args[2].expect_const().to_branch(); assert_eq!(x.layout(), y.layout()); let layout = x.layout(); @@ -143,7 +143,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let total_len = lane_count * 2; - let indexes = idx.iter().map(|idx| idx.unwrap_leaf().to_u32()).collect::>(); + let indexes = idx.iter().map(|idx| idx.to_leaf().to_u32()).collect::>(); for &idx in &indexes { assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len); @@ -961,9 +961,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap(); let ptr_val = ptr.load_scalar(fx); - let alignment = generic_args[3].expect_const().to_value().valtree.unwrap_branch()[0] - .unwrap_leaf() - .to_simd_alignment(); + let alignment = + generic_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment(); let memflags = match alignment { SimdAlign::Unaligned => MemFlags::new().with_notrap(), @@ -1006,15 +1005,6 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let lane_clif_ty = fx.clif_type(val_lane_ty).unwrap(); let ret_lane_layout = fx.layout_of(ret_lane_ty); - let alignment = generic_args[3].expect_const().to_value().valtree.unwrap_branch()[0] - .unwrap_leaf() - .to_simd_alignment(); - - let memflags = match alignment { - SimdAlign::Unaligned => MemFlags::new().with_notrap(), - _ => MemFlags::trusted(), - }; - for lane_idx in 0..ptr_lane_count { let val_lane = val.value_lane(fx, lane_idx).load_scalar(fx); let ptr_lane = ptr.value_lane(fx, lane_idx).load_scalar(fx); @@ -1030,7 +1020,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( fx.bcx.seal_block(if_disabled); fx.bcx.switch_to_block(if_enabled); - let res = fx.bcx.ins().load(lane_clif_ty, memflags, ptr_lane, 0); + let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), ptr_lane, 0); fx.bcx.ins().jump(next, &[res.into()]); fx.bcx.switch_to_block(if_disabled); @@ -1059,9 +1049,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let ret_lane_layout = fx.layout_of(ret_lane_ty); let ptr_val = ptr.load_scalar(fx); - let alignment = generic_args[3].expect_const().to_value().valtree.unwrap_branch()[0] - .unwrap_leaf() - .to_simd_alignment(); + let alignment = + generic_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment(); let memflags = match alignment { SimdAlign::Unaligned => MemFlags::new().with_notrap(), diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs index 2878fa7aa298..65779b38ad1c 100644 --- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs +++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs @@ -10,7 +10,7 @@ //! function u0:22(i64) -> i8, i8 system_v { //! ; symbol _ZN97_$LT$example..IsNotEmpty$u20$as$u20$mini_core..FnOnce$LT$$LP$$RF$$RF$$u5b$u16$u5d$$C$$RP$$GT$$GT$9call_once17hd361e9f5c3d1c4deE //! ; instance Instance { def: Item(DefId(0:42 ~ example[3895]::{impl#0}::call_once)), args: ['{erased}, '{erased}] } -//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: Memory { sized: true }, fields: Arbitrary { offsets: [], memory_index: [] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 12266848898570219025 } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } +//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: Memory { sized: true }, fields: Arbitrary { offsets: [], in_memory_order: [] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 12266848898570219025 } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], in_memory_order: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } //! //! ; kind loc.idx param pass mode ty //! ; ssa _0 (u8, u8) 2b 1 var=(0, 1) @@ -41,7 +41,7 @@ //! ; //! ; _0 = >::call_mut(move _3, copy _2) //! v2 = stack_load.i64 ss0 -//! ; abi: FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: &mut IsNotEmpty, layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(1 bytes)) }) }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } +//! ; abi: FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: &mut IsNotEmpty, layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(1 bytes)) }) }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], in_memory_order: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } //! v3, v4 = call fn0(v1, v2) ; v1 = 1 //! v5 -> v3 //! v6 -> v4 diff --git a/compiler/rustc_codegen_cranelift/triagebot.toml b/compiler/rustc_codegen_cranelift/triagebot.toml index 13da0a87def3..eb0c7b011f60 100644 --- a/compiler/rustc_codegen_cranelift/triagebot.toml +++ b/compiler/rustc_codegen_cranelift/triagebot.toml @@ -2,6 +2,3 @@ # Prevents un-canonicalized issue links (to avoid wrong issues being linked in r-l/rust) [issue-links] - -# Prevents mentions in commits to avoid users being spammed -[no-mentions] diff --git a/compiler/rustc_codegen_gcc/build_system/src/build.rs b/compiler/rustc_codegen_gcc/build_system/src/build.rs index 27476465d740..9b7ee8380ca5 100644 --- a/compiler/rustc_codegen_gcc/build_system/src/build.rs +++ b/compiler/rustc_codegen_gcc/build_system/src/build.rs @@ -111,14 +111,20 @@ pub fn build_sysroot(env: &HashMap, config: &ConfigInfo) -> Resu // Symlink libgccjit.so to sysroot. let lib_path = start_dir.join("sysroot").join("lib"); + let rustlib_target_path = lib_path + .join("rustlib") + .join(&config.host_triple) + .join("codegen-backends") + .join("lib") + .join(&config.target_triple); let libgccjit_path = PathBuf::from(config.gcc_path.as_ref().expect("libgccjit should be set by this point")) .join("libgccjit.so"); - let libgccjit_in_sysroot_path = lib_path.join("libgccjit.so"); + let libgccjit_in_sysroot_path = rustlib_target_path.join("libgccjit.so"); // First remove the file to be able to create the symlink even when the file already exists. let _ = fs::remove_file(&libgccjit_in_sysroot_path); - create_dir(&lib_path)?; - symlink(libgccjit_path, libgccjit_in_sysroot_path) + create_dir(&rustlib_target_path)?; + symlink(libgccjit_path, &libgccjit_in_sysroot_path) .map_err(|error| format!("Cannot create symlink for libgccjit.so: {}", error))?; let library_dir = start_dir.join("sysroot_src").join("library"); diff --git a/compiler/rustc_codegen_gcc/rust-toolchain b/compiler/rustc_codegen_gcc/rust-toolchain index f9645451e964..86ae738d4483 100644 --- a/compiler/rustc_codegen_gcc/rust-toolchain +++ b/compiler/rustc_codegen_gcc/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2025-11-24" +channel = "nightly-2025-12-20" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] diff --git a/compiler/rustc_codegen_gcc/src/builder.rs b/compiler/rustc_codegen_gcc/src/builder.rs index 79228c20d292..3def9a5c015c 100644 --- a/compiler/rustc_codegen_gcc/src/builder.rs +++ b/compiler/rustc_codegen_gcc/src/builder.rs @@ -314,14 +314,12 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> { self.block.get_function() } - fn function_call( + pub fn function_call( &mut self, - func: RValue<'gcc>, + func: Function<'gcc>, args: &[RValue<'gcc>], _funclet: Option<&Funclet>, ) -> RValue<'gcc> { - // TODO(antoyo): remove when the API supports a different type for functions. - let func: Function<'gcc> = self.cx.rvalue_as_function(func); let args = self.check_call("call", func, args); // gccjit requires to use the result of functions, even when it's not used. @@ -514,6 +512,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { type CodegenCx = CodegenCx<'gcc, 'tcx>; fn build(cx: &'a CodegenCx<'gcc, 'tcx>, block: Block<'gcc>) -> Builder<'a, 'gcc, 'tcx> { + *cx.current_func.borrow_mut() = Some(block.get_function()); Builder::with_cx(cx, block) } @@ -1765,6 +1764,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> { // FIXME(antoyo): remove when having a proper API. let gcc_func = unsafe { std::mem::transmute::, Function<'gcc>>(func) }; let call = if self.functions.borrow().values().any(|value| *value == gcc_func) { + // TODO(antoyo): remove when the API supports a different type for functions. + let func: Function<'gcc> = self.cx.rvalue_as_function(func); self.function_call(func, args, funclet) } else { // If it's a not function that was defined, it's a function pointer. diff --git a/compiler/rustc_codegen_gcc/src/context.rs b/compiler/rustc_codegen_gcc/src/context.rs index dbb89a4ff7db..d200d5319a93 100644 --- a/compiler/rustc_codegen_gcc/src/context.rs +++ b/compiler/rustc_codegen_gcc/src/context.rs @@ -92,6 +92,8 @@ pub struct CodegenCx<'gcc, 'tcx> { pub instances: RefCell, LValue<'gcc>>>, /// Cache function instances of monomorphic and polymorphic items pub function_instances: RefCell, Function<'gcc>>>, + /// Cache function instances of intrinsics + pub intrinsic_instances: RefCell, Function<'gcc>>>, /// Cache generated vtables pub vtables: RefCell, Option>), RValue<'gcc>>>, @@ -280,6 +282,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { linkage: Cell::new(FunctionType::Internal), instances: Default::default(), function_instances: Default::default(), + intrinsic_instances: Default::default(), on_stack_params: Default::default(), on_stack_function_params: Default::default(), vtables: Default::default(), @@ -391,17 +394,13 @@ impl<'gcc, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { } fn get_fn(&self, instance: Instance<'tcx>) -> Function<'gcc> { - let func = get_fn(self, instance); - *self.current_func.borrow_mut() = Some(func); - func + get_fn(self, instance) } fn get_fn_addr(&self, instance: Instance<'tcx>) -> RValue<'gcc> { let func_name = self.tcx.symbol_name(instance).name; - let func = if self.intrinsics.borrow().contains_key(func_name) { - self.intrinsics.borrow()[func_name] - } else if let Some(variable) = self.get_declared_value(func_name) { + let func = if let Some(variable) = self.get_declared_value(func_name) { return variable; } else { get_fn(self, instance) diff --git a/compiler/rustc_codegen_gcc/src/declare.rs b/compiler/rustc_codegen_gcc/src/declare.rs index 42d6fb17a88b..e4130b221ee3 100644 --- a/compiler/rustc_codegen_gcc/src/declare.rs +++ b/compiler/rustc_codegen_gcc/src/declare.rs @@ -8,7 +8,6 @@ use rustc_target::callconv::FnAbi; use crate::abi::{FnAbiGcc, FnAbiGccExt}; use crate::context::CodegenCx; -use crate::intrinsic::llvm; impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { pub fn get_or_insert_global( @@ -100,18 +99,14 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { let return_type = self.type_i32(); let variadic = false; self.linkage.set(FunctionType::Exported); - let func = declare_raw_fn( + declare_raw_fn( self, name, callconv, return_type, &[self.type_i32(), const_string], variadic, - ); - // NOTE: it is needed to set the current_func here as well, because get_fn() is not called - // for the main function. - *self.current_func.borrow_mut() = Some(func); - func + ) } pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> Function<'gcc> { @@ -166,19 +161,6 @@ fn declare_raw_fn<'gcc>( param_types: &[Type<'gcc>], variadic: bool, ) -> Function<'gcc> { - if name.starts_with("llvm.") { - let intrinsic = match name { - "llvm.fma.f16" => { - // fma is not a target builtin, but a normal builtin, so we handle it differently - // here. - cx.context.get_builtin_function("fma") - } - _ => llvm::intrinsic(name, cx), - }; - - cx.intrinsics.borrow_mut().insert(name.to_string(), intrinsic); - return intrinsic; - } let func = if cx.functions.borrow().contains_key(name) { cx.functions.borrow()[name] } else { diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/archs.rs b/compiler/rustc_codegen_gcc/src/intrinsic/archs.rs index bb8bcbf66f38..43e7c352c34a 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/archs.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/archs.rs @@ -70,10 +70,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "sve.sm4e" => "__builtin_sve_svsm4e_u32", "sve.sm4ekey" => "__builtin_sve_svsm4ekey_u32", "sve.wrffr" => "__builtin_sve_svwrffr", - "tcancel" => "__builtin_arm_tcancel", - "tcommit" => "__builtin_arm_tcommit", - "tstart" => "__builtin_arm_tstart", - "ttest" => "__builtin_arm_ttest", _ => unimplemented!("***** unsupported LLVM intrinsic {full_name}"), } } @@ -1632,6 +1628,14 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "V6.vabs.f8.128B" => "__builtin_HEXAGON_V6_vabs_f8_128B", "V6.vabs.hf" => "__builtin_HEXAGON_V6_vabs_hf", "V6.vabs.hf.128B" => "__builtin_HEXAGON_V6_vabs_hf_128B", + "V6.vabs.qf16.hf" => "__builtin_HEXAGON_V6_vabs_qf16_hf", + "V6.vabs.qf16.hf.128B" => "__builtin_HEXAGON_V6_vabs_qf16_hf_128B", + "V6.vabs.qf16.qf16" => "__builtin_HEXAGON_V6_vabs_qf16_qf16", + "V6.vabs.qf16.qf16.128B" => "__builtin_HEXAGON_V6_vabs_qf16_qf16_128B", + "V6.vabs.qf32.qf32" => "__builtin_HEXAGON_V6_vabs_qf32_qf32", + "V6.vabs.qf32.qf32.128B" => "__builtin_HEXAGON_V6_vabs_qf32_qf32_128B", + "V6.vabs.qf32.sf" => "__builtin_HEXAGON_V6_vabs_qf32_sf", + "V6.vabs.qf32.sf.128B" => "__builtin_HEXAGON_V6_vabs_qf32_sf_128B", "V6.vabs.sf" => "__builtin_HEXAGON_V6_vabs_sf", "V6.vabs.sf.128B" => "__builtin_HEXAGON_V6_vabs_sf_128B", "V6.vabsb" => "__builtin_HEXAGON_V6_vabsb", @@ -1744,6 +1748,8 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "V6.vaddwsat.128B" => "__builtin_HEXAGON_V6_vaddwsat_128B", "V6.vaddwsat.dv" => "__builtin_HEXAGON_V6_vaddwsat_dv", "V6.vaddwsat.dv.128B" => "__builtin_HEXAGON_V6_vaddwsat_dv_128B", + "V6.valign4" => "__builtin_HEXAGON_V6_valign4", + "V6.valign4.128B" => "__builtin_HEXAGON_V6_valign4_128B", "V6.valignb" => "__builtin_HEXAGON_V6_valignb", "V6.valignb.128B" => "__builtin_HEXAGON_V6_valignb_128B", "V6.valignbi" => "__builtin_HEXAGON_V6_valignbi", @@ -1862,14 +1868,30 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "V6.vcl0w.128B" => "__builtin_HEXAGON_V6_vcl0w_128B", "V6.vcombine" => "__builtin_HEXAGON_V6_vcombine", "V6.vcombine.128B" => "__builtin_HEXAGON_V6_vcombine_128B", + "V6.vconv.bf.qf32" => "__builtin_HEXAGON_V6_vconv_bf_qf32", + "V6.vconv.bf.qf32.128B" => "__builtin_HEXAGON_V6_vconv_bf_qf32_128B", + "V6.vconv.f8.qf16" => "__builtin_HEXAGON_V6_vconv_f8_qf16", + "V6.vconv.f8.qf16.128B" => "__builtin_HEXAGON_V6_vconv_f8_qf16_128B", "V6.vconv.h.hf" => "__builtin_HEXAGON_V6_vconv_h_hf", "V6.vconv.h.hf.128B" => "__builtin_HEXAGON_V6_vconv_h_hf_128B", + "V6.vconv.h.hf.rnd" => "__builtin_HEXAGON_V6_vconv_h_hf_rnd", + "V6.vconv.h.hf.rnd.128B" => "__builtin_HEXAGON_V6_vconv_h_hf_rnd_128B", "V6.vconv.hf.h" => "__builtin_HEXAGON_V6_vconv_hf_h", "V6.vconv.hf.h.128B" => "__builtin_HEXAGON_V6_vconv_hf_h_128B", "V6.vconv.hf.qf16" => "__builtin_HEXAGON_V6_vconv_hf_qf16", "V6.vconv.hf.qf16.128B" => "__builtin_HEXAGON_V6_vconv_hf_qf16_128B", "V6.vconv.hf.qf32" => "__builtin_HEXAGON_V6_vconv_hf_qf32", "V6.vconv.hf.qf32.128B" => "__builtin_HEXAGON_V6_vconv_hf_qf32_128B", + "V6.vconv.qf16.f8" => "__builtin_HEXAGON_V6_vconv_qf16_f8", + "V6.vconv.qf16.f8.128B" => "__builtin_HEXAGON_V6_vconv_qf16_f8_128B", + "V6.vconv.qf16.hf" => "__builtin_HEXAGON_V6_vconv_qf16_hf", + "V6.vconv.qf16.hf.128B" => "__builtin_HEXAGON_V6_vconv_qf16_hf_128B", + "V6.vconv.qf16.qf16" => "__builtin_HEXAGON_V6_vconv_qf16_qf16", + "V6.vconv.qf16.qf16.128B" => "__builtin_HEXAGON_V6_vconv_qf16_qf16_128B", + "V6.vconv.qf32.qf32" => "__builtin_HEXAGON_V6_vconv_qf32_qf32", + "V6.vconv.qf32.qf32.128B" => "__builtin_HEXAGON_V6_vconv_qf32_qf32_128B", + "V6.vconv.qf32.sf" => "__builtin_HEXAGON_V6_vconv_qf32_sf", + "V6.vconv.qf32.sf.128B" => "__builtin_HEXAGON_V6_vconv_qf32_sf_128B", "V6.vconv.sf.qf32" => "__builtin_HEXAGON_V6_vconv_sf_qf32", "V6.vconv.sf.qf32.128B" => "__builtin_HEXAGON_V6_vconv_sf_qf32_128B", "V6.vconv.sf.w" => "__builtin_HEXAGON_V6_vconv_sf_w", @@ -1984,6 +2006,22 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "V6.veqh.or.128B" => "__builtin_HEXAGON_V6_veqh_or_128B", "V6.veqh.xor" => "__builtin_HEXAGON_V6_veqh_xor", "V6.veqh.xor.128B" => "__builtin_HEXAGON_V6_veqh_xor_128B", + "V6.veqhf" => "__builtin_HEXAGON_V6_veqhf", + "V6.veqhf.128B" => "__builtin_HEXAGON_V6_veqhf_128B", + "V6.veqhf.and" => "__builtin_HEXAGON_V6_veqhf_and", + "V6.veqhf.and.128B" => "__builtin_HEXAGON_V6_veqhf_and_128B", + "V6.veqhf.or" => "__builtin_HEXAGON_V6_veqhf_or", + "V6.veqhf.or.128B" => "__builtin_HEXAGON_V6_veqhf_or_128B", + "V6.veqhf.xor" => "__builtin_HEXAGON_V6_veqhf_xor", + "V6.veqhf.xor.128B" => "__builtin_HEXAGON_V6_veqhf_xor_128B", + "V6.veqsf" => "__builtin_HEXAGON_V6_veqsf", + "V6.veqsf.128B" => "__builtin_HEXAGON_V6_veqsf_128B", + "V6.veqsf.and" => "__builtin_HEXAGON_V6_veqsf_and", + "V6.veqsf.and.128B" => "__builtin_HEXAGON_V6_veqsf_and_128B", + "V6.veqsf.or" => "__builtin_HEXAGON_V6_veqsf_or", + "V6.veqsf.or.128B" => "__builtin_HEXAGON_V6_veqsf_or_128B", + "V6.veqsf.xor" => "__builtin_HEXAGON_V6_veqsf_xor", + "V6.veqsf.xor.128B" => "__builtin_HEXAGON_V6_veqsf_xor_128B", "V6.veqw" => "__builtin_HEXAGON_V6_veqw", "V6.veqw.128B" => "__builtin_HEXAGON_V6_veqw_128B", "V6.veqw.and" => "__builtin_HEXAGON_V6_veqw_and", @@ -2096,6 +2134,14 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "V6.vgtw.or.128B" => "__builtin_HEXAGON_V6_vgtw_or_128B", "V6.vgtw.xor" => "__builtin_HEXAGON_V6_vgtw_xor", "V6.vgtw.xor.128B" => "__builtin_HEXAGON_V6_vgtw_xor_128B", + "V6.vilog2.hf" => "__builtin_HEXAGON_V6_vilog2_hf", + "V6.vilog2.hf.128B" => "__builtin_HEXAGON_V6_vilog2_hf_128B", + "V6.vilog2.qf16" => "__builtin_HEXAGON_V6_vilog2_qf16", + "V6.vilog2.qf16.128B" => "__builtin_HEXAGON_V6_vilog2_qf16_128B", + "V6.vilog2.qf32" => "__builtin_HEXAGON_V6_vilog2_qf32", + "V6.vilog2.qf32.128B" => "__builtin_HEXAGON_V6_vilog2_qf32_128B", + "V6.vilog2.sf" => "__builtin_HEXAGON_V6_vilog2_sf", + "V6.vilog2.sf.128B" => "__builtin_HEXAGON_V6_vilog2_sf_128B", "V6.vinsertwr" => "__builtin_HEXAGON_V6_vinsertwr", "V6.vinsertwr.128B" => "__builtin_HEXAGON_V6_vinsertwr_128B", "V6.vlalignb" => "__builtin_HEXAGON_V6_vlalignb", @@ -2350,6 +2396,14 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "V6.vnavgub.128B" => "__builtin_HEXAGON_V6_vnavgub_128B", "V6.vnavgw" => "__builtin_HEXAGON_V6_vnavgw", "V6.vnavgw.128B" => "__builtin_HEXAGON_V6_vnavgw_128B", + "V6.vneg.qf16.hf" => "__builtin_HEXAGON_V6_vneg_qf16_hf", + "V6.vneg.qf16.hf.128B" => "__builtin_HEXAGON_V6_vneg_qf16_hf_128B", + "V6.vneg.qf16.qf16" => "__builtin_HEXAGON_V6_vneg_qf16_qf16", + "V6.vneg.qf16.qf16.128B" => "__builtin_HEXAGON_V6_vneg_qf16_qf16_128B", + "V6.vneg.qf32.qf32" => "__builtin_HEXAGON_V6_vneg_qf32_qf32", + "V6.vneg.qf32.qf32.128B" => "__builtin_HEXAGON_V6_vneg_qf32_qf32_128B", + "V6.vneg.qf32.sf" => "__builtin_HEXAGON_V6_vneg_qf32_sf", + "V6.vneg.qf32.sf.128B" => "__builtin_HEXAGON_V6_vneg_qf32_sf_128B", "V6.vnormamth" => "__builtin_HEXAGON_V6_vnormamth", "V6.vnormamth.128B" => "__builtin_HEXAGON_V6_vnormamth_128B", "V6.vnormamtw" => "__builtin_HEXAGON_V6_vnormamtw", @@ -2684,6 +2738,24 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "iocsrwr.d" => "__builtin_loongarch_iocsrwr_d", "iocsrwr.h" => "__builtin_loongarch_iocsrwr_h", "iocsrwr.w" => "__builtin_loongarch_iocsrwr_w", + "lasx.cast.128" => "__builtin_lasx_cast_128", + "lasx.cast.128.d" => "__builtin_lasx_cast_128_d", + "lasx.cast.128.s" => "__builtin_lasx_cast_128_s", + "lasx.concat.128" => "__builtin_lasx_concat_128", + "lasx.concat.128.d" => "__builtin_lasx_concat_128_d", + "lasx.concat.128.s" => "__builtin_lasx_concat_128_s", + "lasx.extract.128.hi" => "__builtin_lasx_extract_128_hi", + "lasx.extract.128.hi.d" => "__builtin_lasx_extract_128_hi_d", + "lasx.extract.128.hi.s" => "__builtin_lasx_extract_128_hi_s", + "lasx.extract.128.lo" => "__builtin_lasx_extract_128_lo", + "lasx.extract.128.lo.d" => "__builtin_lasx_extract_128_lo_d", + "lasx.extract.128.lo.s" => "__builtin_lasx_extract_128_lo_s", + "lasx.insert.128.hi" => "__builtin_lasx_insert_128_hi", + "lasx.insert.128.hi.d" => "__builtin_lasx_insert_128_hi_d", + "lasx.insert.128.hi.s" => "__builtin_lasx_insert_128_hi_s", + "lasx.insert.128.lo" => "__builtin_lasx_insert_128_lo", + "lasx.insert.128.lo.d" => "__builtin_lasx_insert_128_lo_d", + "lasx.insert.128.lo.s" => "__builtin_lasx_insert_128_lo_s", "lasx.vext2xv.d.b" => "__builtin_lasx_vext2xv_d_b", "lasx.vext2xv.d.h" => "__builtin_lasx_vext2xv_d_h", "lasx.vext2xv.d.w" => "__builtin_lasx_vext2xv_d_w", @@ -4950,8 +5022,20 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "f16x2.to.e5m2x2.rn.relu" => "__nvvm_f16x2_to_e5m2x2_rn_relu", "f2bf16.rn" => "__nvvm_f2bf16_rn", "f2bf16.rn.relu" => "__nvvm_f2bf16_rn_relu", + "f2bf16.rn.relu.satfinite" => "__nvvm_f2bf16_rn_relu_satfinite", + "f2bf16.rn.satfinite" => "__nvvm_f2bf16_rn_satfinite", "f2bf16.rz" => "__nvvm_f2bf16_rz", "f2bf16.rz.relu" => "__nvvm_f2bf16_rz_relu", + "f2bf16.rz.relu.satfinite" => "__nvvm_f2bf16_rz_relu_satfinite", + "f2bf16.rz.satfinite" => "__nvvm_f2bf16_rz_satfinite", + "f2f16.rn" => "__nvvm_f2f16_rn", + "f2f16.rn.relu" => "__nvvm_f2f16_rn_relu", + "f2f16.rn.relu.satfinite" => "__nvvm_f2f16_rn_relu_satfinite", + "f2f16.rn.satfinite" => "__nvvm_f2f16_rn_satfinite", + "f2f16.rz" => "__nvvm_f2f16_rz", + "f2f16.rz.relu" => "__nvvm_f2f16_rz_relu", + "f2f16.rz.relu.satfinite" => "__nvvm_f2f16_rz_relu_satfinite", + "f2f16.rz.satfinite" => "__nvvm_f2f16_rz_satfinite", "f2h.rn" => "__nvvm_f2h_rn", "f2h.rn.ftz" => "__nvvm_f2h_rn_ftz", "f2i.rm" => "__nvvm_f2i_rm", @@ -5035,20 +5119,28 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "ff.to.ue8m0x2.rz.satfinite" => "__nvvm_ff_to_ue8m0x2_rz_satfinite", "ff2bf16x2.rn" => "__nvvm_ff2bf16x2_rn", "ff2bf16x2.rn.relu" => "__nvvm_ff2bf16x2_rn_relu", + "ff2bf16x2.rn.relu.satfinite" => "__nvvm_ff2bf16x2_rn_relu_satfinite", + "ff2bf16x2.rn.satfinite" => "__nvvm_ff2bf16x2_rn_satfinite", "ff2bf16x2.rs" => "__nvvm_ff2bf16x2_rs", "ff2bf16x2.rs.relu" => "__nvvm_ff2bf16x2_rs_relu", "ff2bf16x2.rs.relu.satfinite" => "__nvvm_ff2bf16x2_rs_relu_satfinite", "ff2bf16x2.rs.satfinite" => "__nvvm_ff2bf16x2_rs_satfinite", "ff2bf16x2.rz" => "__nvvm_ff2bf16x2_rz", "ff2bf16x2.rz.relu" => "__nvvm_ff2bf16x2_rz_relu", + "ff2bf16x2.rz.relu.satfinite" => "__nvvm_ff2bf16x2_rz_relu_satfinite", + "ff2bf16x2.rz.satfinite" => "__nvvm_ff2bf16x2_rz_satfinite", "ff2f16x2.rn" => "__nvvm_ff2f16x2_rn", "ff2f16x2.rn.relu" => "__nvvm_ff2f16x2_rn_relu", + "ff2f16x2.rn.relu.satfinite" => "__nvvm_ff2f16x2_rn_relu_satfinite", + "ff2f16x2.rn.satfinite" => "__nvvm_ff2f16x2_rn_satfinite", "ff2f16x2.rs" => "__nvvm_ff2f16x2_rs", "ff2f16x2.rs.relu" => "__nvvm_ff2f16x2_rs_relu", "ff2f16x2.rs.relu.satfinite" => "__nvvm_ff2f16x2_rs_relu_satfinite", "ff2f16x2.rs.satfinite" => "__nvvm_ff2f16x2_rs_satfinite", "ff2f16x2.rz" => "__nvvm_ff2f16x2_rz", "ff2f16x2.rz.relu" => "__nvvm_ff2f16x2_rz_relu", + "ff2f16x2.rz.relu.satfinite" => "__nvvm_ff2f16x2_rz_relu_satfinite", + "ff2f16x2.rz.satfinite" => "__nvvm_ff2f16x2_rz_satfinite", "floor.d" => "__nvvm_floor_d", "floor.f" => "__nvvm_floor_f", "floor.ftz.f" => "__nvvm_floor_ftz_f", @@ -5942,6 +6034,8 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "altivec.vupklsb" => "__builtin_altivec_vupklsb", "altivec.vupklsh" => "__builtin_altivec_vupklsh", "altivec.vupklsw" => "__builtin_altivec_vupklsw", + "amo.ldat" => "__builtin_amo_ldat", + "amo.lwat" => "__builtin_amo_lwat", "bcdadd" => "__builtin_ppc_bcdadd", "bcdadd.p" => "__builtin_ppc_bcdadd_p", "bcdcopysign" => "__builtin_ppc_bcdcopysign", @@ -6202,6 +6296,7 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "vsx.xvminsp" => "__builtin_vsx_xvminsp", "vsx.xvredp" => "__builtin_vsx_xvredp", "vsx.xvresp" => "__builtin_vsx_xvresp", + "vsx.xvrlw" => "__builtin_vsx_xvrlw", "vsx.xvrsqrtedp" => "__builtin_vsx_xvrsqrtedp", "vsx.xvrsqrtesp" => "__builtin_vsx_xvrsqrtesp", "vsx.xvtdivdp" => "__builtin_vsx_xvtdivdp", @@ -10158,24 +10253,16 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "stui" => "__builtin_ia32_stui", "subborrow.u32" => "__builtin_ia32_subborrow_u32", "subborrow.u64" => "__builtin_ia32_subborrow_u64", - "t2rpntlvwz0" => "__builtin_ia32_t2rpntlvwz0", "t2rpntlvwz0rs" => "__builtin_ia32_t2rpntlvwz0rs", "t2rpntlvwz0rst1" => "__builtin_ia32_t2rpntlvwz0rst1", - "t2rpntlvwz0t1" => "__builtin_ia32_t2rpntlvwz0t1", - "t2rpntlvwz1" => "__builtin_ia32_t2rpntlvwz1", "t2rpntlvwz1rs" => "__builtin_ia32_t2rpntlvwz1rs", "t2rpntlvwz1rst1" => "__builtin_ia32_t2rpntlvwz1rst1", - "t2rpntlvwz1t1" => "__builtin_ia32_t2rpntlvwz1t1", "tbm.bextri.u32" => "__builtin_ia32_bextri_u32", "tbm.bextri.u64" => "__builtin_ia32_bextri_u64", "tcmmimfp16ps" => "__builtin_ia32_tcmmimfp16ps", "tcmmimfp16ps.internal" => "__builtin_ia32_tcmmimfp16ps_internal", "tcmmrlfp16ps" => "__builtin_ia32_tcmmrlfp16ps", "tcmmrlfp16ps.internal" => "__builtin_ia32_tcmmrlfp16ps_internal", - "tconjtcmmimfp16ps" => "__builtin_ia32_tconjtcmmimfp16ps", - "tconjtcmmimfp16ps.internal" => "__builtin_ia32_tconjtcmmimfp16ps_internal", - "tconjtfp16" => "__builtin_ia32_tconjtfp16", - "tconjtfp16.internal" => "__builtin_ia32_tconjtfp16_internal", "tcvtrowd2ps" => "__builtin_ia32_tcvtrowd2ps", "tcvtrowd2ps.internal" => "__builtin_ia32_tcvtrowd2ps_internal", "tcvtrowps2bf16h" => "__builtin_ia32_tcvtrowps2bf16h", @@ -10225,18 +10312,6 @@ fn map_arch_intrinsic(full_name: &str) -> &'static str { "tmmultf32ps" => "__builtin_ia32_tmmultf32ps", "tmmultf32ps.internal" => "__builtin_ia32_tmmultf32ps_internal", "tpause" => "__builtin_ia32_tpause", - "ttcmmimfp16ps" => "__builtin_ia32_ttcmmimfp16ps", - "ttcmmimfp16ps.internal" => "__builtin_ia32_ttcmmimfp16ps_internal", - "ttcmmrlfp16ps" => "__builtin_ia32_ttcmmrlfp16ps", - "ttcmmrlfp16ps.internal" => "__builtin_ia32_ttcmmrlfp16ps_internal", - "ttdpbf16ps" => "__builtin_ia32_ttdpbf16ps", - "ttdpbf16ps.internal" => "__builtin_ia32_ttdpbf16ps_internal", - "ttdpfp16ps" => "__builtin_ia32_ttdpfp16ps", - "ttdpfp16ps.internal" => "__builtin_ia32_ttdpfp16ps_internal", - "ttmmultf32ps" => "__builtin_ia32_ttmmultf32ps", - "ttmmultf32ps.internal" => "__builtin_ia32_ttmmultf32ps_internal", - "ttransposed" => "__builtin_ia32_ttransposed", - "ttransposed.internal" => "__builtin_ia32_ttransposed_internal", "umonitor" => "__builtin_ia32_umonitor", "umwait" => "__builtin_ia32_umwait", "urdmsr" => "__builtin_ia32_urdmsr", diff --git a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs index d2714ba7914f..36ea76cbc51a 100644 --- a/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs +++ b/compiler/rustc_codegen_gcc/src/intrinsic/mod.rs @@ -9,7 +9,7 @@ use gccjit::Type; use gccjit::{ComparisonOp, Function, FunctionType, RValue, ToRValue, UnaryOp}; #[cfg(feature = "master")] use rustc_abi::ExternAbi; -use rustc_abi::{BackendRepr, HasDataLayout}; +use rustc_abi::{BackendRepr, HasDataLayout, WrappingRange}; use rustc_codegen_ssa::MemFlags; use rustc_codegen_ssa::base::wants_msvc_seh; use rustc_codegen_ssa::common::IntPredicate; @@ -20,19 +20,15 @@ use rustc_codegen_ssa::mir::place::{PlaceRef, PlaceValue}; use rustc_codegen_ssa::traits::MiscCodegenMethods; use rustc_codegen_ssa::traits::{ ArgAbiBuilderMethods, BaseTypeCodegenMethods, BuilderMethods, ConstCodegenMethods, - IntrinsicCallBuilderMethods, + IntrinsicCallBuilderMethods, LayoutTypeCodegenMethods, }; use rustc_middle::bug; -#[cfg(feature = "master")] -use rustc_middle::ty::layout::FnAbiOf; -use rustc_middle::ty::layout::LayoutOf; +use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; use rustc_middle::ty::{self, Instance, Ty}; use rustc_span::{Span, Symbol, sym}; use rustc_target::callconv::{ArgAbi, PassMode}; -#[cfg(feature = "master")] -use crate::abi::FnAbiGccExt; -use crate::abi::GccType; +use crate::abi::{FnAbiGccExt, GccType}; use crate::builder::Builder; use crate::common::{SignType, TypeReflection}; use crate::context::CodegenCx; @@ -609,6 +605,94 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc Ok(()) } + fn codegen_llvm_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, Self::Value>], + is_cleanup: bool, + ) -> Self::Value { + let func = if let Some(&func) = self.intrinsic_instances.borrow().get(&instance) { + func + } else { + let sym = self.tcx.symbol_name(instance).name; + + let func = if let Some(func) = self.intrinsics.borrow().get(sym) { + *func + } else { + self.linkage.set(FunctionType::Extern); + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); + let fn_ty = fn_abi.gcc_type(self); + + let func = match sym { + "llvm.fma.f16" => { + // fma is not a target builtin, but a normal builtin, so we handle it differently + // here. + self.context.get_builtin_function("fma") + } + _ => llvm::intrinsic(sym, self), + }; + + self.intrinsics.borrow_mut().insert(sym.to_string(), func); + + self.on_stack_function_params + .borrow_mut() + .insert(func, fn_ty.on_stack_param_indices); + #[cfg(feature = "master")] + for fn_attr in fn_ty.fn_attributes { + func.add_attribute(fn_attr); + } + + crate::attributes::from_fn_attrs(self, func, instance); + + func + }; + + self.intrinsic_instances.borrow_mut().insert(instance, func); + + func + }; + let fn_ptr = func.get_address(None); + let fn_ty = fn_ptr.get_type(); + + let mut llargs = vec![]; + + for arg in args { + match arg.val { + OperandValue::ZeroSized => {} + OperandValue::Immediate(_) => llargs.push(arg.immediate()), + OperandValue::Pair(a, b) => { + llargs.push(a); + llargs.push(b); + } + OperandValue::Ref(op_place_val) => { + let mut llval = op_place_val.llval; + // We can't use `PlaceRef::load` here because the argument + // may have a type we don't treat as immediate, but the ABI + // used for this call is passing it by-value. In that case, + // the load would just produce `OperandValue::Ref` instead + // of the `OperandValue::Immediate` we need for the call. + llval = self.load(self.backend_type(arg.layout), llval, op_place_val.align); + if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr { + if scalar.is_bool() { + self.range_metadata(llval, WrappingRange { start: 0, end: 1 }); + } + // We store bools as `i8` so we need to truncate to `i1`. + llval = self.to_immediate_scalar(llval, scalar); + } + llargs.push(llval); + } + } + } + + // FIXME directly use the llvm intrinsic adjustment functions here + let llret = self.call(fn_ty, None, None, fn_ptr, &llargs, None, None); + if is_cleanup { + self.apply_attrs_to_cleanup_callsite(llret); + } + + llret + } + fn abort(&mut self) { let func = self.context.get_builtin_function("abort"); let func: RValue<'gcc> = unsafe { std::mem::transmute(func) }; diff --git a/compiler/rustc_codegen_gcc/tests/failing-ui-tests.txt b/compiler/rustc_codegen_gcc/tests/failing-ui-tests.txt index 2380bd0fc137..99a80fa7e4f4 100644 --- a/compiler/rustc_codegen_gcc/tests/failing-ui-tests.txt +++ b/compiler/rustc_codegen_gcc/tests/failing-ui-tests.txt @@ -88,3 +88,12 @@ tests/ui/test-attrs/test-panic-while-printing.rs tests/ui/thir-print/offset_of.rs tests/ui/iterators/rangefrom-overflow-debug.rs tests/ui/iterators/rangefrom-overflow-overflow-checks.rs +tests/ui/iterators/iter-filter-count-debug-check.rs +tests/ui/eii/codegen_single_crate.rs +tests/ui/eii/codegen_cross_crate.rs +tests/ui/eii/default/local_crate.rs +tests/ui/eii/multiple_impls.rs +tests/ui/eii/default/call_default.rs +tests/ui/eii/same-symbol.rs +tests/ui/eii/privacy1.rs +tests/ui/eii/default/call_impl.rs diff --git a/compiler/rustc_codegen_gcc/triagebot.toml b/compiler/rustc_codegen_gcc/triagebot.toml index 13da0a87def3..eb0c7b011f60 100644 --- a/compiler/rustc_codegen_gcc/triagebot.toml +++ b/compiler/rustc_codegen_gcc/triagebot.toml @@ -2,6 +2,3 @@ # Prevents un-canonicalized issue links (to avoid wrong issues being linked in r-l/rust) [issue-links] - -# Prevents mentions in commits to avoid users being spammed -[no-mentions] diff --git a/compiler/rustc_codegen_llvm/Cargo.toml b/compiler/rustc_codegen_llvm/Cargo.toml index 076ec5e59eb6..9741436aacb7 100644 --- a/compiler/rustc_codegen_llvm/Cargo.toml +++ b/compiler/rustc_codegen_llvm/Cargo.toml @@ -14,7 +14,7 @@ bitflags = "2.4.1" gimli = "0.31" itertools = "0.12" libc = "0.2" -libloading = { version = "0.9.0", optional = true } +libloading = { version = "0.9.0" } measureme = "12.0.1" object = { version = "0.37.0", default-features = false, features = ["std", "read"] } rustc-demangle = "0.1.21" @@ -38,8 +38,6 @@ rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } rustc_symbol_mangling = { path = "../rustc_symbol_mangling" } rustc_target = { path = "../rustc_target" } -serde = { version = "1", features = ["derive"] } -serde_json = "1" smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } tracing = "0.1" # tidy-alphabetical-end @@ -47,7 +45,7 @@ tracing = "0.1" [features] # tidy-alphabetical-start check_only = ["rustc_llvm/check_only"] -llvm_enzyme = ["dep:libloading"] +llvm_enzyme = [] llvm_offload = [] # tidy-alphabetical-end diff --git a/compiler/rustc_codegen_llvm/messages.ftl b/compiler/rustc_codegen_llvm/messages.ftl index 0e7b00d0bcb7..a637ae8184b4 100644 --- a/compiler/rustc_codegen_llvm/messages.ftl +++ b/compiler/rustc_codegen_llvm/messages.ftl @@ -1,9 +1,10 @@ +codegen_llvm_autodiff_component_unavailable = failed to load our autodiff backend. Did you install it via rustup? + codegen_llvm_autodiff_without_enable = using the autodiff feature requires -Z autodiff=Enable codegen_llvm_autodiff_without_lto = using the autodiff feature requires setting `lto="fat"` in your Cargo.toml codegen_llvm_copy_bitcode = failed to copy bitcode to object file: {$err} - codegen_llvm_fixed_x18_invalid_arch = the `-Zfixed-x18` flag is not supported on the `{$arch}` architecture codegen_llvm_from_llvm_diag = {$message} @@ -18,7 +19,12 @@ codegen_llvm_lto_bitcode_from_rlib = failed to get bitcode from object file for codegen_llvm_mismatch_data_layout = data-layout for target `{$rustc_target}`, `{$rustc_layout}`, differs from LLVM target's `{$llvm_target}` default layout, `{$llvm_layout}` -codegen_llvm_offload_without_enable = using the offload feature requires -Z offload=Enable +codegen_llvm_offload_bundleimages_failed = call to BundleImages failed, `host.out` was not created +codegen_llvm_offload_embed_failed = call to EmbedBufferInModule failed, `host.o` was not created +codegen_llvm_offload_no_abs_path = using the `-Z offload=Host=/absolute/path/to/host.out` flag requires an absolute path +codegen_llvm_offload_no_host_out = using the `-Z offload=Host=/absolute/path/to/host.out` flag must point to a `host.out` file +codegen_llvm_offload_nonexisting = the given path/file to `host.out` does not exist. Did you forget to run the device compilation first? +codegen_llvm_offload_without_enable = using the offload feature requires -Z offload= codegen_llvm_offload_without_fat_lto = using the offload feature requires -C lto=fat codegen_llvm_parse_bitcode = failed to parse bitcode for LTO module diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs index 314e64272ffe..52c2ca7b1696 100644 --- a/compiler/rustc_codegen_llvm/src/back/lto.rs +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -528,7 +528,6 @@ fn thin_lto( } } -#[cfg(feature = "llvm_enzyme")] pub(crate) fn enable_autodiff_settings(ad: &[config::AutoDiff]) { let mut enzyme = llvm::EnzymeWrapper::get_instance(); diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs index 18da945f5a31..e9306e050803 100644 --- a/compiler/rustc_codegen_llvm/src/back/write.rs +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -568,8 +568,7 @@ pub(crate) unsafe fn llvm_optimize( // FIXME(ZuseZ4): In a future update we could figure out how to only optimize individual functions getting // differentiated. - let consider_ad = - cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable); + let consider_ad = config.autodiff.contains(&config::AutoDiff::Enable); let run_enzyme = autodiff_stage == AutodiffStage::DuringAD; let print_before_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModBefore); let print_after_enzyme = config.autodiff.contains(&config::AutoDiff::PrintModAfter); @@ -704,11 +703,9 @@ pub(crate) unsafe fn llvm_optimize( llvm::set_value_name(new_fn, &name); } - if cgcx.target_is_like_gpu && config.offload.contains(&config::Offload::Enable) { + if cgcx.target_is_like_gpu && config.offload.contains(&config::Offload::Device) { let cx = SimpleCx::new(module.module_llvm.llmod(), module.module_llvm.llcx, cgcx.pointer_size); - // For now we only support up to 10 kernels named kernel_0 ... kernel_9, a follow-up PR is - // introducing a proper offload intrinsic to solve this limitation. for func in cx.get_functions() { let offload_kernel = "offload-kernel"; if attributes::has_string_attr(func, offload_kernel) { @@ -777,12 +774,77 @@ pub(crate) unsafe fn llvm_optimize( ) }; - if cgcx.target_is_like_gpu && config.offload.contains(&config::Offload::Enable) { + if cgcx.target_is_like_gpu && config.offload.contains(&config::Offload::Device) { + let device_path = cgcx.output_filenames.path(OutputType::Object); + let device_dir = device_path.parent().unwrap(); + let device_out = device_dir.join("host.out"); + let device_out_c = path_to_c_string(device_out.as_path()); unsafe { - llvm::LLVMRustBundleImages(module.module_llvm.llmod(), module.module_llvm.tm.raw()); + // 1) Bundle device module into offload image host.out (device TM) + let ok = llvm::LLVMRustBundleImages( + module.module_llvm.llmod(), + module.module_llvm.tm.raw(), + device_out_c.as_ptr(), + ); + if !ok || !device_out.exists() { + dcx.emit_err(crate::errors::OffloadBundleImagesFailed); + } } } + // This assumes that we previously compiled our kernels for a gpu target, which created a + // `host.out` artifact. The user is supposed to provide us with a path to this artifact, we + // don't need any other artifacts from the previous run. We will embed this artifact into our + // LLVM-IR host module, to create a `host.o` ObjectFile, which we will write to disk. + // The last, not yet automated steps uses the `clang-linker-wrapper` to process `host.o`. + if !cgcx.target_is_like_gpu { + if let Some(device_path) = config + .offload + .iter() + .find_map(|o| if let config::Offload::Host(path) = o { Some(path) } else { None }) + { + let device_pathbuf = PathBuf::from(device_path); + if device_pathbuf.is_relative() { + dcx.emit_err(crate::errors::OffloadWithoutAbsPath); + } else if device_pathbuf + .file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n != "host.out") + { + dcx.emit_err(crate::errors::OffloadWrongFileName); + } else if !device_pathbuf.exists() { + dcx.emit_err(crate::errors::OffloadNonexistingPath); + } + let host_path = cgcx.output_filenames.path(OutputType::Object); + let host_dir = host_path.parent().unwrap(); + let out_obj = host_dir.join("host.o"); + let host_out_c = path_to_c_string(device_pathbuf.as_path()); + + // 2) Finalize host: lib.bc + host.out -> host.o (host TM) + // We create a full clone of our LLVM host module, since we will embed the device IR + // into it, and this might break caching or incremental compilation otherwise. + let llmod2 = llvm::LLVMCloneModule(module.module_llvm.llmod()); + let ok = + unsafe { llvm::LLVMRustOffloadEmbedBufferInModule(llmod2, host_out_c.as_ptr()) }; + if !ok { + dcx.emit_err(crate::errors::OffloadEmbedFailed); + } + write_output_file( + dcx, + module.module_llvm.tm.raw(), + config.no_builtins, + llmod2, + &out_obj, + None, + llvm::FileType::ObjectFile, + &cgcx.prof, + true, + ); + // We ignore cgcx.save_temps here and unconditionally always keep our `host.out` artifact. + // Otherwise, recompiling the host code would fail since we deleted that device artifact + // in the previous host compilation, which would be confusing at best. + } + } result.into_result().unwrap_or_else(|()| llvm_err(dcx, LlvmError::RunLlvmPasses)) } @@ -819,8 +881,7 @@ pub(crate) fn optimize( // If we know that we will later run AD, then we disable vectorization and loop unrolling. // Otherwise we pretend AD is already done and run the normal opt pipeline (=PostAD). - let consider_ad = - cfg!(feature = "llvm_enzyme") && config.autodiff.contains(&config::AutoDiff::Enable); + let consider_ad = config.autodiff.contains(&config::AutoDiff::Enable); let autodiff_stage = if consider_ad { AutodiffStage::PreAD } else { AutodiffStage::PostAD }; // The embedded bitcode is used to run LTO/ThinLTO. // The bitcode obtained during the `codegen` phase is no longer suitable for performing LTO. @@ -1075,7 +1136,7 @@ pub(crate) fn codegen( EmitObj::None => {} } - record_llvm_cgu_instructions_stats(&cgcx.prof, llmod); + record_llvm_cgu_instructions_stats(&cgcx.prof, &module.name, llmod); } // `.dwo` files are only emitted if: @@ -1282,22 +1343,11 @@ fn record_artifact_size( } } -fn record_llvm_cgu_instructions_stats(prof: &SelfProfilerRef, llmod: &llvm::Module) { +fn record_llvm_cgu_instructions_stats(prof: &SelfProfilerRef, name: &str, llmod: &llvm::Module) { if !prof.enabled() { return; } - let raw_stats = - llvm::build_string(|s| unsafe { llvm::LLVMRustModuleInstructionStats(llmod, s) }) - .expect("cannot get module instruction stats"); - - #[derive(serde::Deserialize)] - struct InstructionsStats { - module: String, - total: u64, - } - - let InstructionsStats { module, total } = - serde_json::from_str(&raw_stats).expect("cannot parse llvm cgu instructions stats"); - prof.artifact_size("cgu_instructions", module, total); + let total = unsafe { llvm::LLVMRustModuleInstructionStats(llmod) }; + prof.artifact_size("cgu_instructions", name, total); } diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs index 6cbddfec4631..388118f9b4f1 100644 --- a/compiler/rustc_codegen_llvm/src/base.rs +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -23,13 +23,14 @@ use rustc_middle::dep_graph; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrs, SanitizerFnAttrs}; use rustc_middle::mir::mono::Visibility; use rustc_middle::ty::TyCtxt; -use rustc_session::config::DebugInfo; +use rustc_session::config::{DebugInfo, Offload}; use rustc_span::Symbol; use rustc_target::spec::SanitizerSet; use super::ModuleLlvm; use crate::attributes; use crate::builder::Builder; +use crate::builder::gpu_offload::OffloadGlobals; use crate::context::CodegenCx; use crate::llvm::{self, Value}; @@ -85,6 +86,19 @@ pub(crate) fn compile_codegen_unit( let llvm_module = ModuleLlvm::new(tcx, cgu_name.as_str()); { let mut cx = CodegenCx::new(tcx, cgu, &llvm_module); + + // Declare and store globals shared by all offload kernels + // + // These globals are left in the LLVM-IR host module so all kernels can access them. + // They are necessary for correct offload execution. We do this here to simplify the + // `offload` intrinsic, avoiding the need for tracking whether it's the first + // intrinsic call or not. + let has_host_offload = + cx.sess().opts.unstable_opts.offload.iter().any(|o| matches!(o, Offload::Host(_))); + if has_host_offload && !cx.sess().target.is_like_gpu { + cx.offload_globals.replace(Some(OffloadGlobals::declare(&cx))); + } + let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx); for &(mono_item, data) in &mono_items { mono_item.predefine::>( diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 9a9e4dcdbb3c..7a49ba64029e 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -1705,7 +1705,7 @@ impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { ret.expect("LLVM does not have support for catchret") } - fn check_call<'b>( + pub(crate) fn check_call<'b>( &mut self, typ: &str, fn_ty: &'ll Type, diff --git a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs index 5d1ddd057d88..046501d08c48 100644 --- a/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs +++ b/compiler/rustc_codegen_llvm/src/builder/gpu_offload.rs @@ -2,17 +2,76 @@ use std::ffi::CString; use llvm::Linkage::*; use rustc_abi::Align; -use rustc_codegen_ssa::traits::BaseTypeCodegenMethods; +use rustc_codegen_ssa::traits::{BaseTypeCodegenMethods, BuilderMethods}; use rustc_middle::ty::offload_meta::OffloadMetadata; -use crate::builder::SBuilder; +use crate::builder::Builder; +use crate::common::CodegenCx; use crate::llvm::AttributePlace::Function; -use crate::llvm::{self, BasicBlock, Linkage, Type, Value}; +use crate::llvm::{self, Linkage, Type, Value}; use crate::{SimpleCx, attributes}; +// LLVM kernel-independent globals required for offloading +pub(crate) struct OffloadGlobals<'ll> { + pub launcher_fn: &'ll llvm::Value, + pub launcher_ty: &'ll llvm::Type, + + pub bin_desc: &'ll llvm::Type, + + pub kernel_args_ty: &'ll llvm::Type, + + pub offload_entry_ty: &'ll llvm::Type, + + pub begin_mapper: &'ll llvm::Value, + pub end_mapper: &'ll llvm::Value, + pub mapper_fn_ty: &'ll llvm::Type, + + pub ident_t_global: &'ll llvm::Value, + + pub register_lib: &'ll llvm::Value, + pub unregister_lib: &'ll llvm::Value, + pub init_rtls: &'ll llvm::Value, +} + +impl<'ll> OffloadGlobals<'ll> { + pub(crate) fn declare(cx: &CodegenCx<'ll, '_>) -> Self { + let (launcher_fn, launcher_ty) = generate_launcher(cx); + let kernel_args_ty = KernelArgsTy::new_decl(cx); + let offload_entry_ty = TgtOffloadEntry::new_decl(cx); + let (begin_mapper, _, end_mapper, mapper_fn_ty) = gen_tgt_data_mappers(cx); + let ident_t_global = generate_at_one(cx); + + let tptr = cx.type_ptr(); + let ti32 = cx.type_i32(); + let tgt_bin_desc_ty = vec![ti32, tptr, tptr, tptr]; + let bin_desc = cx.type_named_struct("struct.__tgt_bin_desc"); + cx.set_struct_body(bin_desc, &tgt_bin_desc_ty, false); + + let register_lib = declare_offload_fn(&cx, "__tgt_register_lib", mapper_fn_ty); + let unregister_lib = declare_offload_fn(&cx, "__tgt_unregister_lib", mapper_fn_ty); + let init_ty = cx.type_func(&[], cx.type_void()); + let init_rtls = declare_offload_fn(cx, "__tgt_init_all_rtls", init_ty); + + OffloadGlobals { + launcher_fn, + launcher_ty, + bin_desc, + kernel_args_ty, + offload_entry_ty, + begin_mapper, + end_mapper, + mapper_fn_ty, + ident_t_global, + register_lib, + unregister_lib, + init_rtls, + } + } +} + // ; Function Attrs: nounwind // declare i32 @__tgt_target_kernel(ptr, i64, i32, i32, ptr, ptr) #2 -fn generate_launcher<'ll>(cx: &'ll SimpleCx<'_>) -> (&'ll llvm::Value, &'ll llvm::Type) { +fn generate_launcher<'ll>(cx: &CodegenCx<'ll, '_>) -> (&'ll llvm::Value, &'ll llvm::Type) { let tptr = cx.type_ptr(); let ti64 = cx.type_i64(); let ti32 = cx.type_i32(); @@ -30,7 +89,7 @@ fn generate_launcher<'ll>(cx: &'ll SimpleCx<'_>) -> (&'ll llvm::Value, &'ll llvm // @1 = private unnamed_addr constant %struct.ident_t { i32 0, i32 2, i32 0, i32 22, ptr @0 }, align 8 // FIXME(offload): @0 should include the file name (e.g. lib.rs) in which the function to be // offloaded was defined. -fn generate_at_one<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll llvm::Value { +pub(crate) fn generate_at_one<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll llvm::Value { let unknown_txt = ";unknown;unknown;0;0;;"; let c_entry_name = CString::new(unknown_txt).unwrap(); let c_val = c_entry_name.as_bytes_with_nul(); @@ -68,7 +127,7 @@ pub(crate) struct TgtOffloadEntry { } impl TgtOffloadEntry { - pub(crate) fn new_decl<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll llvm::Type { + pub(crate) fn new_decl<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll llvm::Type { let offload_entry_ty = cx.type_named_struct("struct.__tgt_offload_entry"); let tptr = cx.type_ptr(); let ti64 = cx.type_i64(); @@ -82,7 +141,7 @@ impl TgtOffloadEntry { } fn new<'ll>( - cx: &'ll SimpleCx<'_>, + cx: &CodegenCx<'ll, '_>, region_id: &'ll Value, llglobal: &'ll Value, ) -> [&'ll Value; 9] { @@ -126,7 +185,7 @@ impl KernelArgsTy { const OFFLOAD_VERSION: u64 = 3; const FLAGS: u64 = 0; const TRIPCOUNT: u64 = 0; - fn new_decl<'ll>(cx: &'ll SimpleCx<'_>) -> &'ll Type { + fn new_decl<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Type { let kernel_arguments_ty = cx.type_named_struct("struct.__tgt_kernel_arguments"); let tptr = cx.type_ptr(); let ti64 = cx.type_i64(); @@ -140,8 +199,8 @@ impl KernelArgsTy { kernel_arguments_ty } - fn new<'ll>( - cx: &'ll SimpleCx<'_>, + fn new<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, num_args: u64, memtransfer_types: &'ll Value, geps: [&'ll Value; 3], @@ -171,7 +230,8 @@ impl KernelArgsTy { } // Contains LLVM values needed to manage offloading for a single kernel. -pub(crate) struct OffloadKernelData<'ll> { +#[derive(Copy, Clone)] +pub(crate) struct OffloadKernelGlobals<'ll> { pub offload_sizes: &'ll llvm::Value, pub memtransfer_types: &'ll llvm::Value, pub region_id: &'ll llvm::Value, @@ -179,7 +239,7 @@ pub(crate) struct OffloadKernelData<'ll> { } fn gen_tgt_data_mappers<'ll>( - cx: &'ll SimpleCx<'_>, + cx: &CodegenCx<'ll, '_>, ) -> (&'ll llvm::Value, &'ll llvm::Value, &'ll llvm::Value, &'ll llvm::Type) { let tptr = cx.type_ptr(); let ti64 = cx.type_i64(); @@ -241,12 +301,18 @@ pub(crate) fn add_global<'ll>( // mapped to/from the gpu. It also returns a region_id with the name of this kernel, to be // concatenated into the list of region_ids. pub(crate) fn gen_define_handling<'ll>( - cx: &SimpleCx<'ll>, - offload_entry_ty: &'ll llvm::Type, + cx: &CodegenCx<'ll, '_>, metadata: &[OffloadMetadata], - types: &[&Type], - symbol: &str, -) -> OffloadKernelData<'ll> { + types: &[&'ll Type], + symbol: String, + offload_globals: &OffloadGlobals<'ll>, +) -> OffloadKernelGlobals<'ll> { + if let Some(entry) = cx.offload_kernel_cache.borrow().get(&symbol) { + return *entry; + } + + let offload_entry_ty = offload_globals.offload_entry_ty; + // It seems like non-pointer values are automatically mapped. So here, we focus on pointer (or // reference) types. let ptr_meta = types.iter().zip(metadata).filter_map(|(&x, meta)| match cx.type_kind(x) { @@ -272,9 +338,9 @@ pub(crate) fn gen_define_handling<'ll>( let name = format!(".{symbol}.region_id"); let initializer = cx.get_const_i8(0); - let region_id = add_unnamed_global(&cx, &name, initializer, WeakAnyLinkage); + let region_id = add_global(&cx, &name, initializer, WeakAnyLinkage); - let c_entry_name = CString::new(symbol).unwrap(); + let c_entry_name = CString::new(symbol.clone()).unwrap(); let c_val = c_entry_name.as_bytes_with_nul(); let offload_entry_name = format!(".offloading.entry_name.{symbol}"); @@ -298,11 +364,16 @@ pub(crate) fn gen_define_handling<'ll>( let c_section_name = CString::new("llvm_offload_entries").unwrap(); llvm::set_section(offload_entry, &c_section_name); - OffloadKernelData { offload_sizes, memtransfer_types, region_id, offload_entry } + let result = + OffloadKernelGlobals { offload_sizes, memtransfer_types, region_id, offload_entry }; + + cx.offload_kernel_cache.borrow_mut().insert(symbol, result); + + result } fn declare_offload_fn<'ll>( - cx: &'ll SimpleCx<'_>, + cx: &CodegenCx<'ll, '_>, name: &str, ty: &'ll llvm::Type, ) -> &'ll llvm::Value { @@ -335,28 +406,28 @@ fn declare_offload_fn<'ll>( // 4. set insert point after kernel call. // 5. generate all the GEPS and stores, to be used in 6) // 6. generate __tgt_target_data_end calls to move data from the GPU -pub(crate) fn gen_call_handling<'ll>( - cx: &SimpleCx<'ll>, - bb: &BasicBlock, - offload_data: &OffloadKernelData<'ll>, +pub(crate) fn gen_call_handling<'ll, 'tcx>( + builder: &mut Builder<'_, 'll, 'tcx>, + offload_data: &OffloadKernelGlobals<'ll>, args: &[&'ll Value], types: &[&Type], metadata: &[OffloadMetadata], + offload_globals: &OffloadGlobals<'ll>, ) { - let OffloadKernelData { offload_sizes, offload_entry, memtransfer_types, region_id } = + let cx = builder.cx; + let OffloadKernelGlobals { offload_sizes, offload_entry, memtransfer_types, region_id } = offload_data; - let (tgt_decl, tgt_target_kernel_ty) = generate_launcher(&cx); + + let tgt_decl = offload_globals.launcher_fn; + let tgt_target_kernel_ty = offload_globals.launcher_ty; + // %struct.__tgt_bin_desc = type { i32, ptr, ptr, ptr } - let tptr = cx.type_ptr(); - let ti32 = cx.type_i32(); - let tgt_bin_desc_ty = vec![ti32, tptr, tptr, tptr]; - let tgt_bin_desc = cx.type_named_struct("struct.__tgt_bin_desc"); - cx.set_struct_body(tgt_bin_desc, &tgt_bin_desc_ty, false); + let tgt_bin_desc = offload_globals.bin_desc; - let tgt_kernel_decl = KernelArgsTy::new_decl(&cx); - let (begin_mapper_decl, _, end_mapper_decl, fn_ty) = gen_tgt_data_mappers(&cx); - - let mut builder = SBuilder::build(cx, bb); + let tgt_kernel_decl = offload_globals.kernel_args_ty; + let begin_mapper_decl = offload_globals.begin_mapper; + let end_mapper_decl = offload_globals.end_mapper; + let fn_ty = offload_globals.mapper_fn_ty; let num_args = types.len() as u64; let ip = unsafe { llvm::LLVMRustGetInsertPoint(&builder.llbuilder) }; @@ -378,9 +449,8 @@ pub(crate) fn gen_call_handling<'ll>( // Step 0) // %struct.__tgt_bin_desc = type { i32, ptr, ptr, ptr } // %6 = alloca %struct.__tgt_bin_desc, align 8 - let llfn = unsafe { llvm::LLVMGetBasicBlockParent(bb) }; unsafe { - llvm::LLVMRustPositionBuilderPastAllocas(&builder.llbuilder, llfn); + llvm::LLVMRustPositionBuilderPastAllocas(&builder.llbuilder, builder.llfn()); } let tgt_bin_desc_alloca = builder.direct_alloca(tgt_bin_desc, Align::EIGHT, "EmptyDesc"); @@ -413,16 +483,16 @@ pub(crate) fn gen_call_handling<'ll>( } let mapper_fn_ty = cx.type_func(&[cx.type_ptr()], cx.type_void()); - let register_lib_decl = declare_offload_fn(&cx, "__tgt_register_lib", mapper_fn_ty); - let unregister_lib_decl = declare_offload_fn(&cx, "__tgt_unregister_lib", mapper_fn_ty); + let register_lib_decl = offload_globals.register_lib; + let unregister_lib_decl = offload_globals.unregister_lib; let init_ty = cx.type_func(&[], cx.type_void()); - let init_rtls_decl = declare_offload_fn(cx, "__tgt_init_all_rtls", init_ty); + let init_rtls_decl = offload_globals.init_rtls; // FIXME(offload): Later we want to add them to the wrapper code, rather than our main function. // call void @__tgt_register_lib(ptr noundef %6) - builder.call(mapper_fn_ty, register_lib_decl, &[tgt_bin_desc_alloca], None); + builder.call(mapper_fn_ty, None, None, register_lib_decl, &[tgt_bin_desc_alloca], None, None); // call void @__tgt_init_all_rtls() - builder.call(init_ty, init_rtls_decl, &[], None); + builder.call(init_ty, None, None, init_rtls_decl, &[], None, None); for i in 0..num_args { let idx = cx.get_const_i32(i); @@ -437,15 +507,15 @@ pub(crate) fn gen_call_handling<'ll>( // For now we have a very simplistic indexing scheme into our // offload_{baseptrs,ptrs,sizes}. We will probably improve this along with our gpu frontend pr. - fn get_geps<'a, 'll>( - builder: &mut SBuilder<'a, 'll>, - cx: &'ll SimpleCx<'ll>, + fn get_geps<'ll, 'tcx>( + builder: &mut Builder<'_, 'll, 'tcx>, ty: &'ll Type, ty2: &'ll Type, a1: &'ll Value, a2: &'ll Value, a4: &'ll Value, ) -> [&'ll Value; 3] { + let cx = builder.cx; let i32_0 = cx.get_const_i32(0); let gep1 = builder.inbounds_gep(ty, a1, &[i32_0, i32_0]); @@ -454,9 +524,8 @@ pub(crate) fn gen_call_handling<'ll>( [gep1, gep2, gep3] } - fn generate_mapper_call<'a, 'll>( - builder: &mut SBuilder<'a, 'll>, - cx: &'ll SimpleCx<'ll>, + fn generate_mapper_call<'ll, 'tcx>( + builder: &mut Builder<'_, 'll, 'tcx>, geps: [&'ll Value; 3], o_type: &'ll Value, fn_to_call: &'ll Value, @@ -464,20 +533,20 @@ pub(crate) fn gen_call_handling<'ll>( num_args: u64, s_ident_t: &'ll Value, ) { + let cx = builder.cx; let nullptr = cx.const_null(cx.type_ptr()); let i64_max = cx.get_const_i64(u64::MAX); let num_args = cx.get_const_i32(num_args); let args = vec![s_ident_t, i64_max, num_args, geps[0], geps[1], geps[2], o_type, nullptr, nullptr]; - builder.call(fn_ty, fn_to_call, &args, None); + builder.call(fn_ty, None, None, fn_to_call, &args, None, None); } // Step 2) - let s_ident_t = generate_at_one(&cx); - let geps = get_geps(&mut builder, &cx, ty, ty2, a1, a2, a4); + let s_ident_t = offload_globals.ident_t_global; + let geps = get_geps(builder, ty, ty2, a1, a2, a4); generate_mapper_call( - &mut builder, - &cx, + builder, geps, memtransfer_types, begin_mapper_decl, @@ -504,14 +573,13 @@ pub(crate) fn gen_call_handling<'ll>( region_id, a5, ]; - builder.call(tgt_target_kernel_ty, tgt_decl, &args, None); + builder.call(tgt_target_kernel_ty, None, None, tgt_decl, &args, None, None); // %41 = call i32 @__tgt_target_kernel(ptr @1, i64 -1, i32 2097152, i32 256, ptr @.kernel_1.region_id, ptr %kernel_args) // Step 4) - let geps = get_geps(&mut builder, &cx, ty, ty2, a1, a2, a4); + let geps = get_geps(builder, ty, ty2, a1, a2, a4); generate_mapper_call( - &mut builder, - &cx, + builder, geps, memtransfer_types, end_mapper_decl, @@ -520,7 +588,5 @@ pub(crate) fn gen_call_handling<'ll>( s_ident_t, ); - builder.call(mapper_fn_ty, unregister_lib_decl, &[tgt_bin_desc_alloca], None); - - drop(builder); + builder.call(mapper_fn_ty, None, None, unregister_lib_decl, &[tgt_bin_desc_alloca], None, None); } diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index 6caf60e3cc41..07b04863af6b 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -35,6 +35,7 @@ use smallvec::SmallVec; use crate::abi::to_llvm_calling_convention; use crate::back::write::to_llvm_code_model; +use crate::builder::gpu_offload::{OffloadGlobals, OffloadKernelGlobals}; use crate::callee::get_fn; use crate::debuginfo::metadata::apply_vcall_visibility_metadata; use crate::llvm::{self, Metadata, MetadataKindId, Module, Type, Value}; @@ -100,6 +101,8 @@ pub(crate) struct FullCx<'ll, 'tcx> { /// Cache instances of monomorphic and polymorphic items pub instances: RefCell, &'ll Value>>, + /// Cache instances of intrinsics + pub intrinsic_instances: RefCell, &'ll Value>>, /// Cache generated vtables pub vtables: RefCell, Option>), &'ll Value>>, /// Cache of constant strings, @@ -156,6 +159,12 @@ pub(crate) struct FullCx<'ll, 'tcx> { /// Cache of Objective-C selector references pub objc_selrefs: RefCell>, + + /// Globals shared by the offloading runtime + pub offload_globals: RefCell>>, + + /// Cache of kernel-specific globals + pub offload_kernel_cache: RefCell>>, } fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { @@ -620,6 +629,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { tls_model, codegen_unit, instances: Default::default(), + intrinsic_instances: Default::default(), vtables: Default::default(), const_str_cache: Default::default(), const_globals: Default::default(), @@ -639,6 +649,8 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { objc_class_t: Cell::new(None), objc_classrefs: Default::default(), objc_selrefs: Default::default(), + offload_globals: Default::default(), + offload_kernel_cache: Default::default(), }, PhantomData, ) diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/di_builder.rs b/compiler/rustc_codegen_llvm/src/debuginfo/di_builder.rs index 5d874631ca1a..0b5ef6c68740 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/di_builder.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/di_builder.rs @@ -1,8 +1,37 @@ +use std::ptr; + use libc::c_uint; use rustc_abi::Align; use crate::llvm::debuginfo::DIBuilder; -use crate::llvm::{self, ToLlvmBool}; +use crate::llvm::{self, Module, ToLlvmBool}; + +/// Owning pointer to a `DIBuilder<'ll>` that will dispose of the builder +/// when dropped. Use `.as_ref()` to get the underlying `&DIBuilder` +/// needed for debuginfo FFI calls. +pub(crate) struct DIBuilderBox<'ll> { + raw: ptr::NonNull>, +} + +impl<'ll> DIBuilderBox<'ll> { + pub(crate) fn new(llmod: &'ll Module) -> Self { + let raw = unsafe { llvm::LLVMCreateDIBuilder(llmod) }; + let raw = ptr::NonNull::new(raw).unwrap(); + Self { raw } + } + + pub(crate) fn as_ref(&self) -> &DIBuilder<'ll> { + // SAFETY: This is an owning pointer, so `&DIBuilder` is valid + // for as long as `&self` is. + unsafe { self.raw.as_ref() } + } +} + +impl<'ll> Drop for DIBuilderBox<'ll> { + fn drop(&mut self) { + unsafe { llvm::LLVMDisposeDIBuilder(self.raw) }; + } +} /// Extension trait for defining safe wrappers and helper methods on /// `&DIBuilder<'ll>`, without requiring it to be defined in the same crate. diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs index c2c55ee2b64f..1c63bbcd1710 100644 --- a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -38,8 +38,9 @@ use self::namespace::mangled_name_of_instance; use self::utils::{DIB, create_DIArray, is_node_local_to_unit}; use crate::builder::Builder; use crate::common::{AsCCharPtr, CodegenCx}; +use crate::debuginfo::di_builder::DIBuilderBox; use crate::llvm::debuginfo::{ - DIArray, DIBuilderBox, DIFile, DIFlags, DILexicalBlock, DILocation, DISPFlags, DIScope, + DIArray, DIFile, DIFlags, DILexicalBlock, DILocation, DISPFlags, DIScope, DITemplateTypeParameter, DIType, DIVariable, }; use crate::llvm::{self, Value}; diff --git a/compiler/rustc_codegen_llvm/src/errors.rs b/compiler/rustc_codegen_llvm/src/errors.rs index dd9fde0b08c6..c73140e041b6 100644 --- a/compiler/rustc_codegen_llvm/src/errors.rs +++ b/compiler/rustc_codegen_llvm/src/errors.rs @@ -32,6 +32,10 @@ impl Diagnostic<'_, G> for ParseTargetMachineConfig<'_> { } } +#[derive(Diagnostic)] +#[diag(codegen_llvm_autodiff_component_unavailable)] +pub(crate) struct AutoDiffComponentUnavailable; + #[derive(Diagnostic)] #[diag(codegen_llvm_autodiff_without_lto)] pub(crate) struct AutoDiffWithoutLto; @@ -48,6 +52,26 @@ pub(crate) struct OffloadWithoutEnable; #[diag(codegen_llvm_offload_without_fat_lto)] pub(crate) struct OffloadWithoutFatLTO; +#[derive(Diagnostic)] +#[diag(codegen_llvm_offload_no_abs_path)] +pub(crate) struct OffloadWithoutAbsPath; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_offload_no_host_out)] +pub(crate) struct OffloadWrongFileName; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_offload_nonexisting)] +pub(crate) struct OffloadNonexistingPath; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_offload_bundleimages_failed)] +pub(crate) struct OffloadBundleImagesFailed; + +#[derive(Diagnostic)] +#[diag(codegen_llvm_offload_embed_failed)] +pub(crate) struct OffloadEmbedFailed; + #[derive(Diagnostic)] #[diag(codegen_llvm_lto_bitcode_from_rlib)] pub(crate) struct LtoBitcodeFromRlib { diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 82615d4a160e..b4057eea735e 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -1,7 +1,11 @@ use std::assert_matches::assert_matches; use std::cmp::Ordering; +use std::ffi::c_uint; +use std::ptr; -use rustc_abi::{Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size}; +use rustc_abi::{ + Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size, WrappingRange, +}; use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh}; use rustc_codegen_ssa::codegen_attrs::autodiff_attrs; use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; @@ -26,8 +30,9 @@ use tracing::debug; use crate::abi::FnAbiLlvmExt; use crate::builder::Builder; use crate::builder::autodiff::{adjust_activity_to_abi, generate_enzyme_call}; -use crate::builder::gpu_offload::TgtOffloadEntry; +use crate::builder::gpu_offload::{gen_call_handling, gen_define_handling}; use crate::context::CodegenCx; +use crate::declare::declare_raw_fn; use crate::errors::{ AutoDiffWithoutEnable, AutoDiffWithoutLto, OffloadWithoutEnable, OffloadWithoutFatLTO, }; @@ -202,13 +207,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { return Ok(()); } sym::offload => { - if !tcx - .sess - .opts - .unstable_opts - .offload - .contains(&rustc_session::config::Offload::Enable) - { + if tcx.sess.opts.unstable_opts.offload.is_empty() { let _ = tcx.dcx().emit_almost_fatal(OffloadWithoutEnable); } @@ -351,7 +350,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { _ => bug!(), }; let ptr = args[0].immediate(); - let locality = fn_args.const_at(1).to_value().valtree.unwrap_leaf().to_i32(); + let locality = fn_args.const_at(1).to_leaf().to_i32(); self.call_intrinsic( "llvm.prefetch", &[self.val_ty(ptr)], @@ -639,6 +638,99 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { Ok(()) } + fn codegen_llvm_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, Self::Value>], + is_cleanup: bool, + ) -> Self::Value { + let tcx = self.tcx(); + + // FIXME remove usage of fn_abi + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); + assert!(!fn_abi.ret.is_indirect()); + let fn_ty = fn_abi.llvm_type(self); + + let fn_ptr = if let Some(&llfn) = self.intrinsic_instances.borrow().get(&instance) { + llfn + } else { + let sym = tcx.symbol_name(instance).name; + + // FIXME use get_intrinsic + let llfn = if let Some(llfn) = self.get_declared_value(sym) { + llfn + } else { + // Function addresses in Rust are never significant, allowing functions to + // be merged. + let llfn = declare_raw_fn( + self, + sym, + fn_abi.llvm_cconv(self), + llvm::UnnamedAddr::Global, + llvm::Visibility::Default, + fn_ty, + ); + fn_abi.apply_attrs_llfn(self, llfn, Some(instance)); + + llfn + }; + + self.intrinsic_instances.borrow_mut().insert(instance, llfn); + + llfn + }; + + let mut llargs = vec![]; + + for arg in args { + match arg.val { + OperandValue::ZeroSized => {} + OperandValue::Immediate(_) => llargs.push(arg.immediate()), + OperandValue::Pair(a, b) => { + llargs.push(a); + llargs.push(b); + } + OperandValue::Ref(op_place_val) => { + let mut llval = op_place_val.llval; + // We can't use `PlaceRef::load` here because the argument + // may have a type we don't treat as immediate, but the ABI + // used for this call is passing it by-value. In that case, + // the load would just produce `OperandValue::Ref` instead + // of the `OperandValue::Immediate` we need for the call. + llval = self.load(self.backend_type(arg.layout), llval, op_place_val.align); + if let BackendRepr::Scalar(scalar) = arg.layout.backend_repr { + if scalar.is_bool() { + self.range_metadata(llval, WrappingRange { start: 0, end: 1 }); + } + // We store bools as `i8` so we need to truncate to `i1`. + llval = self.to_immediate_scalar(llval, scalar); + } + llargs.push(llval); + } + } + } + + debug!("call intrinsic {:?} with args ({:?})", instance, llargs); + let args = self.check_call("call", fn_ty, fn_ptr, &llargs); + let llret = unsafe { + llvm::LLVMBuildCallWithOperandBundles( + self.llbuilder, + fn_ty, + fn_ptr, + args.as_ptr() as *const &llvm::Value, + args.len() as c_uint, + ptr::dangling(), + 0, + c"".as_ptr(), + ) + }; + if is_cleanup { + self.apply_attrs_to_cleanup_callsite(llret); + } + + llret + } + fn abort(&mut self) { self.call_intrinsic("llvm.trap", &[], &[]); } @@ -1295,8 +1387,6 @@ fn codegen_offload<'ll, 'tcx>( let args = get_args_from_tuple(bx, args[1], fn_target); let target_symbol = symbol_name_for_instance_in_crate(tcx, fn_target, LOCAL_CRATE); - let offload_entry_ty = TgtOffloadEntry::new_decl(&cx); - let sig = tcx.fn_sig(fn_target.def_id()).skip_binder().skip_binder(); let inputs = sig.inputs(); @@ -1304,17 +1394,16 @@ fn codegen_offload<'ll, 'tcx>( let types = inputs.iter().map(|ty| cx.layout_of(*ty).llvm_type(cx)).collect::>(); - let offload_data = crate::builder::gpu_offload::gen_define_handling( - cx, - offload_entry_ty, - &metadata, - &types, - &target_symbol, - ); - - // FIXME(Sa4dUs): pass the original builder once we separate kernel launch logic from globals - let bb = unsafe { llvm::LLVMGetInsertBlock(bx.llbuilder) }; - crate::builder::gpu_offload::gen_call_handling(cx, bb, &offload_data, &args, &types, &metadata); + let offload_globals_ref = cx.offload_globals.borrow(); + let offload_globals = match offload_globals_ref.as_ref() { + Some(globals) => globals, + None => { + // Offload is not initialized, cannot continue + return; + } + }; + let offload_data = gen_define_handling(&cx, &metadata, &types, target_symbol, offload_globals); + gen_call_handling(bx, &offload_data, &args, &types, &metadata, offload_globals); } fn get_args_from_tuple<'ll, 'tcx>( @@ -1536,7 +1625,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( } if name == sym::simd_shuffle_const_generic { - let idx = fn_args[2].expect_const().to_value().valtree.unwrap_branch(); + let idx = fn_args[2].expect_const().to_branch(); let n = idx.len() as u64; let (out_len, out_ty) = require_simd!(ret_ty, SimdReturn); @@ -1555,7 +1644,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( .iter() .enumerate() .map(|(arg_idx, val)| { - let idx = val.unwrap_leaf().to_i32(); + let idx = val.to_leaf().to_i32(); if idx >= i32::try_from(total_len).unwrap() { bx.sess().dcx().emit_err(InvalidMonomorphization::SimdIndexOutOfBounds { span, @@ -1967,9 +2056,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // those lanes whose `mask` bit is enabled. // The memory addresses corresponding to the “off” lanes are not accessed. - let alignment = fn_args[3].expect_const().to_value().valtree.unwrap_branch()[0] - .unwrap_leaf() - .to_simd_alignment(); + let alignment = fn_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment(); // The element type of the "mask" argument must be a signed integer type of any width let mask_ty = in_ty; @@ -2062,9 +2149,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( // those lanes whose `mask` bit is enabled. // The memory addresses corresponding to the “off” lanes are not accessed. - let alignment = fn_args[3].expect_const().to_value().valtree.unwrap_branch()[0] - .unwrap_leaf() - .to_simd_alignment(); + let alignment = fn_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment(); // The element type of the "mask" argument must be a signed integer type of any width let mask_ty = in_ty; diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs index c51b334d95e1..095274744993 100644 --- a/compiler/rustc_codegen_llvm/src/lib.rs +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -5,7 +5,6 @@ //! This API is completely unstable and subject to change. // tidy-alphabetical-start -#![cfg_attr(bootstrap, feature(slice_as_array))] #![feature(assert_matches)] #![feature(extern_types)] #![feature(file_buffered)] @@ -13,6 +12,7 @@ #![feature(impl_trait_in_assoc_type)] #![feature(iter_intersperse)] #![feature(macro_derive)] +#![feature(once_cell_try)] #![feature(trim_prefix_suffix)] #![feature(try_blocks)] // tidy-alphabetical-end @@ -241,13 +241,17 @@ impl CodegenBackend for LlvmCodegenBackend { fn init(&self, sess: &Session) { llvm_util::init(sess); // Make sure llvm is inited - #[cfg(feature = "llvm_enzyme")] + // autodiff is based on Enzyme, a library which we might not have available, when it was + // neither build, nor downloaded via rustup. If autodiff is used, but not available we emit + // an early error here and abort compilation. { use rustc_session::config::AutoDiff; use crate::back::lto::enable_autodiff_settings; if sess.opts.unstable_opts.autodiff.contains(&AutoDiff::Enable) { - drop(llvm::EnzymeWrapper::get_or_init(&sess.opts.sysroot)); + if let Err(_) = llvm::EnzymeWrapper::get_or_init(&sess.opts.sysroot) { + sess.dcx().emit_fatal(crate::errors::AutoDiffComponentUnavailable); + } enable_autodiff_settings(&sess.opts.unstable_opts.autodiff); } } diff --git a/compiler/rustc_codegen_llvm/src/llvm/conversions.rs b/compiler/rustc_codegen_llvm/src/llvm/conversions.rs index 9e9f9339ade8..3bc790ca7cff 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/conversions.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/conversions.rs @@ -1,6 +1,6 @@ //! Conversions from backend-independent data types to/from LLVM FFI types. -use rustc_codegen_ssa::common::{AtomicRmwBinOp, IntPredicate, RealPredicate}; +use rustc_codegen_ssa::common::{AtomicRmwBinOp, IntPredicate, RealPredicate, TypeKind}; use rustc_middle::ty::AtomicOrdering; use rustc_session::config::DebugInfo; use rustc_target::spec::SymbolVisibility; @@ -9,10 +9,22 @@ use crate::llvm; /// Helper trait for converting backend-independent types to LLVM-specific /// types, for FFI purposes. +/// +/// FIXME(#147327): These trait/method names were chosen to avoid churn in +/// existing code, but are not great and could probably be made clearer. pub(crate) trait FromGeneric { fn from_generic(other: T) -> Self; } +/// Helper trait for converting LLVM-specific types to backend-independent +/// types, for FFI purposes. +/// +/// FIXME(#147327): These trait/method names were chosen to avoid churn in +/// existing code, but are not great and could probably be made clearer. +pub(crate) trait ToGeneric { + fn to_generic(&self) -> T; +} + impl FromGeneric for llvm::Visibility { fn from_generic(visibility: SymbolVisibility) -> Self { match visibility { @@ -113,3 +125,29 @@ impl FromGeneric for llvm::debuginfo::DebugEmissionKind { } } } + +impl ToGeneric for llvm::TypeKind { + fn to_generic(&self) -> TypeKind { + match self { + Self::Void => TypeKind::Void, + Self::Half => TypeKind::Half, + Self::Float => TypeKind::Float, + Self::Double => TypeKind::Double, + Self::X86_FP80 => TypeKind::X86_FP80, + Self::FP128 => TypeKind::FP128, + Self::PPC_FP128 => TypeKind::PPC_FP128, + Self::Label => TypeKind::Label, + Self::Integer => TypeKind::Integer, + Self::Function => TypeKind::Function, + Self::Struct => TypeKind::Struct, + Self::Array => TypeKind::Array, + Self::Pointer => TypeKind::Pointer, + Self::Vector => TypeKind::Vector, + Self::Metadata => TypeKind::Metadata, + Self::Token => TypeKind::Token, + Self::ScalableVector => TypeKind::ScalableVector, + Self::BFloat => TypeKind::BFloat, + Self::X86_AMX => TypeKind::X86_AMX, + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs index 28923bf2743e..b11310b970d0 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/enzyme_ffi.rs @@ -86,10 +86,8 @@ pub(crate) enum LLVMRustVerifierFailureAction { LLVMReturnStatusAction = 2, } -#[cfg(feature = "llvm_enzyme")] pub(crate) use self::Enzyme_AD::*; -#[cfg(feature = "llvm_enzyme")] pub(crate) mod Enzyme_AD { use std::ffi::{c_char, c_void}; use std::sync::{Mutex, MutexGuard, OnceLock}; @@ -199,15 +197,13 @@ pub(crate) mod Enzyme_AD { /// Safe to call multiple times - subsequent calls are no-ops due to OnceLock. pub(crate) fn get_or_init( sysroot: &rustc_session::config::Sysroot, - ) -> MutexGuard<'static, Self> { - ENZYME_INSTANCE - .get_or_init(|| { - Self::call_dynamic(sysroot) - .unwrap_or_else(|e| bug!("failed to load Enzyme: {e}")) - .into() - }) - .lock() - .unwrap() + ) -> Result, Box> { + let mtx: &'static Mutex = ENZYME_INSTANCE.get_or_try_init(|| { + let w = Self::call_dynamic(sysroot)?; + Ok::<_, Box>(Mutex::new(w)) + })?; + + Ok(mtx.lock().unwrap()) } /// Get the EnzymeWrapper instance. Panics if not initialized. @@ -452,147 +448,6 @@ pub(crate) mod Enzyme_AD { } } -#[cfg(not(feature = "llvm_enzyme"))] -pub(crate) use self::Fallback_AD::*; - -#[cfg(not(feature = "llvm_enzyme"))] -pub(crate) mod Fallback_AD { - #![allow(unused_variables)] - - use std::ffi::c_void; - use std::sync::{Mutex, MutexGuard}; - - use libc::c_char; - use rustc_codegen_ssa::back::write::CodegenContext; - use rustc_codegen_ssa::traits::WriteBackendMethods; - - use super::{CConcreteType, CTypeTreeRef, Context, EnzymeTypeTree}; - - pub(crate) struct EnzymeWrapper { - pub registerEnzymeAndPassPipeline: *const c_void, - } - - impl EnzymeWrapper { - pub(crate) fn get_or_init( - _sysroot: &rustc_session::config::Sysroot, - ) -> MutexGuard<'static, Self> { - unimplemented!("Enzyme not available: build with llvm_enzyme feature") - } - - pub(crate) fn init<'a, B: WriteBackendMethods>( - _cgcx: &'a CodegenContext, - ) -> &'static Mutex { - unimplemented!("Enzyme not available: build with llvm_enzyme feature") - } - - pub(crate) fn get_instance() -> MutexGuard<'static, Self> { - unimplemented!("Enzyme not available: build with llvm_enzyme feature") - } - - pub(crate) fn new_type_tree(&self) -> CTypeTreeRef { - unimplemented!() - } - - pub(crate) fn new_type_tree_ct( - &self, - t: CConcreteType, - ctx: &Context, - ) -> *mut EnzymeTypeTree { - unimplemented!() - } - - pub(crate) fn new_type_tree_tr(&self, tree: CTypeTreeRef) -> CTypeTreeRef { - unimplemented!() - } - - pub(crate) fn free_type_tree(&self, tree: CTypeTreeRef) { - unimplemented!() - } - - pub(crate) fn merge_type_tree(&self, tree1: CTypeTreeRef, tree2: CTypeTreeRef) -> bool { - unimplemented!() - } - - pub(crate) fn tree_only_eq(&self, tree: CTypeTreeRef, num: i64) { - unimplemented!() - } - - pub(crate) fn tree_data0_eq(&self, tree: CTypeTreeRef) { - unimplemented!() - } - - pub(crate) fn shift_indicies_eq( - &self, - tree: CTypeTreeRef, - data_layout: *const c_char, - offset: i64, - max_size: i64, - add_offset: u64, - ) { - unimplemented!() - } - - pub(crate) fn tree_insert_eq( - &self, - tree: CTypeTreeRef, - indices: *const i64, - len: usize, - ct: CConcreteType, - ctx: &Context, - ) { - unimplemented!() - } - - pub(crate) fn tree_to_string(&self, tree: *mut EnzymeTypeTree) -> *const c_char { - unimplemented!() - } - - pub(crate) fn tree_to_string_free(&self, ch: *const c_char) { - unimplemented!() - } - - pub(crate) fn get_max_type_depth(&self) -> usize { - unimplemented!() - } - - pub(crate) fn set_inline(&mut self, val: bool) { - unimplemented!() - } - - pub(crate) fn set_print_perf(&mut self, print: bool) { - unimplemented!() - } - - pub(crate) fn set_print_activity(&mut self, print: bool) { - unimplemented!() - } - - pub(crate) fn set_print_type(&mut self, print: bool) { - unimplemented!() - } - - pub(crate) fn set_print_type_fun(&mut self, fun_name: &str) { - unimplemented!() - } - - pub(crate) fn set_print(&mut self, print: bool) { - unimplemented!() - } - - pub(crate) fn set_strict_aliasing(&mut self, strict: bool) { - unimplemented!() - } - - pub(crate) fn set_loose_types(&mut self, loose: bool) { - unimplemented!() - } - - pub(crate) fn set_rust_rules(&mut self, val: bool) { - unimplemented!() - } - } -} - impl TypeTree { pub(crate) fn new() -> TypeTree { let wrapper = EnzymeWrapper::get_instance(); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 2f53a1995248..ff91b2b691e3 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -363,33 +363,6 @@ pub(crate) enum TypeKind { X86_AMX = 19, } -impl TypeKind { - pub(crate) fn to_generic(self) -> rustc_codegen_ssa::common::TypeKind { - use rustc_codegen_ssa::common::TypeKind as Common; - match self { - Self::Void => Common::Void, - Self::Half => Common::Half, - Self::Float => Common::Float, - Self::Double => Common::Double, - Self::X86_FP80 => Common::X86_FP80, - Self::FP128 => Common::FP128, - Self::PPC_FP128 => Common::PPC_FP128, - Self::Label => Common::Label, - Self::Integer => Common::Integer, - Self::Function => Common::Function, - Self::Struct => Common::Struct, - Self::Array => Common::Array, - Self::Pointer => Common::Pointer, - Self::Vector => Common::Vector, - Self::Metadata => Common::Metadata, - Self::Token => Common::Token, - Self::ScalableVector => Common::ScalableVector, - Self::BFloat => Common::BFloat, - Self::X86_AMX => Common::X86_AMX, - } - } -} - /// LLVMAtomicRmwBinOp #[derive(Copy, Clone)] #[repr(C)] @@ -738,12 +711,9 @@ unsafe extern "C" { pub(crate) type DiagnosticHandlerTy = unsafe extern "C" fn(&DiagnosticInfo, *mut c_void); pub(crate) mod debuginfo { - use std::ptr; - use bitflags::bitflags; use super::{InvariantOpaque, Metadata}; - use crate::llvm::{self, Module}; /// Opaque target type for references to an LLVM debuginfo builder. /// @@ -756,33 +726,6 @@ pub(crate) mod debuginfo { #[repr(C)] pub(crate) struct DIBuilder<'ll>(InvariantOpaque<'ll>); - /// Owning pointer to a `DIBuilder<'ll>` that will dispose of the builder - /// when dropped. Use `.as_ref()` to get the underlying `&DIBuilder` - /// needed for debuginfo FFI calls. - pub(crate) struct DIBuilderBox<'ll> { - raw: ptr::NonNull>, - } - - impl<'ll> DIBuilderBox<'ll> { - pub(crate) fn new(llmod: &'ll Module) -> Self { - let raw = unsafe { llvm::LLVMCreateDIBuilder(llmod) }; - let raw = ptr::NonNull::new(raw).unwrap(); - Self { raw } - } - - pub(crate) fn as_ref(&self) -> &DIBuilder<'ll> { - // SAFETY: This is an owning pointer, so `&DIBuilder` is valid - // for as long as `&self` is. - unsafe { self.raw.as_ref() } - } - } - - impl<'ll> Drop for DIBuilderBox<'ll> { - fn drop(&mut self) { - unsafe { llvm::LLVMDisposeDIBuilder(self.raw) }; - } - } - pub(crate) type DIDescriptor = Metadata; pub(crate) type DILocation = Metadata; pub(crate) type DIScope = DIDescriptor; @@ -1723,7 +1666,15 @@ mod Offload { use super::*; unsafe extern "C" { /// Processes the module and writes it in an offload compatible way into a "host.out" file. - pub(crate) fn LLVMRustBundleImages<'a>(M: &'a Module, TM: &'a TargetMachine) -> bool; + pub(crate) fn LLVMRustBundleImages<'a>( + M: &'a Module, + TM: &'a TargetMachine, + host_out: *const c_char, + ) -> bool; + pub(crate) unsafe fn LLVMRustOffloadEmbedBufferInModule<'a>( + _M: &'a Module, + _host_out: *const c_char, + ) -> bool; pub(crate) fn LLVMRustOffloadMapper<'a>(OldFn: &'a Value, NewFn: &'a Value); } } @@ -1737,7 +1688,17 @@ mod Offload_fallback { /// Processes the module and writes it in an offload compatible way into a "host.out" file. /// Marked as unsafe to match the real offload wrapper which is unsafe due to FFI. #[allow(unused_unsafe)] - pub(crate) unsafe fn LLVMRustBundleImages<'a>(_M: &'a Module, _TM: &'a TargetMachine) -> bool { + pub(crate) unsafe fn LLVMRustBundleImages<'a>( + _M: &'a Module, + _TM: &'a TargetMachine, + _host_out: *const c_char, + ) -> bool { + unimplemented!("This rustc version was not built with LLVM Offload support!"); + } + pub(crate) unsafe fn LLVMRustOffloadEmbedBufferInModule<'a>( + _M: &'a Module, + _host_out: *const c_char, + ) -> bool { unimplemented!("This rustc version was not built with LLVM Offload support!"); } #[allow(unused_unsafe)] @@ -2493,7 +2454,7 @@ unsafe extern "C" { pub(crate) fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; pub(crate) fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); pub(crate) fn LLVMRustModuleCost(M: &Module) -> u64; - pub(crate) fn LLVMRustModuleInstructionStats(M: &Module, Str: &RustString); + pub(crate) fn LLVMRustModuleInstructionStats(M: &Module) -> u64; pub(crate) fn LLVMRustThinLTOBufferCreate( M: &Module, diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs index 682484595a89..01181ce26184 100644 --- a/compiler/rustc_codegen_llvm/src/llvm_util.rs +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -266,6 +266,10 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option Some(LLVMFeature::new("hasleoncasa")), s => Some(LLVMFeature::new(s)), }, + Arch::Wasm32 | Arch::Wasm64 => match s { + "gc" if major < 22 => None, + s => Some(LLVMFeature::new(s)), + }, Arch::X86 | Arch::X86_64 => { match s { "sse4.2" => Some(LLVMFeature::with_dependencies( @@ -360,25 +364,26 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) { let target_abi = &sess.target.options.abi; let target_pointer_width = sess.target.pointer_width; let version = get_version(); - let lt_20_1_1 = version < (20, 1, 1); - let lt_21_0_0 = version < (21, 0, 0); + let (major, _, _) = version; cfg.has_reliable_f16 = match (target_arch, target_os) { - // LLVM crash without neon (fixed in llvm20) + // LLVM crash without neon (fixed in LLVM 20.1.1) (Arch::AArch64, _) - if !cfg.target_features.iter().any(|f| f.as_str() == "neon") && lt_20_1_1 => + if !cfg.target_features.iter().any(|f| f.as_str() == "neon") + && version < (20, 1, 1) => { false } // Unsupported (Arch::Arm64EC, _) => false, // Selection failure (fixed in llvm21) - (Arch::S390x, _) if lt_21_0_0 => false, + (Arch::S390x, _) if major < 21 => false, // MinGW ABI bugs (Arch::X86_64, Os::Windows) if *target_env == Env::Gnu && *target_abi != Abi::Llvm => false, // Infinite recursion (Arch::CSky, _) => false, - (Arch::Hexagon, _) if lt_21_0_0 => false, // (fixed in llvm21) + (Arch::Hexagon, _) if major < 21 => false, // (fixed in llvm21) + (Arch::LoongArch32 | Arch::LoongArch64, _) if major < 21 => false, // (fixed in llvm21) (Arch::PowerPC | Arch::PowerPC64, _) => false, (Arch::Sparc | Arch::Sparc64, _) => false, (Arch::Wasm32 | Arch::Wasm64, _) => false, @@ -389,15 +394,15 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) { }; cfg.has_reliable_f128 = match (target_arch, target_os) { + // Unsupported https://github.com/llvm/llvm-project/issues/121122 + (Arch::AmdGpu, _) => false, // Unsupported (Arch::Arm64EC, _) => false, - // Selection bug (fixed in llvm20) - (Arch::Mips64 | Arch::Mips64r6, _) if lt_20_1_1 => false, + // Selection bug (fixed in LLVM 20.1.0) + (Arch::Mips64 | Arch::Mips64r6, _) if version < (20, 1, 0) => false, // Selection bug . This issue is closed // but basic math still does not work. (Arch::Nvptx64, _) => false, - // Unsupported https://github.com/llvm/llvm-project/issues/121122 - (Arch::AmdGpu, _) => false, // ABI bugs et al. (full // list at ) (Arch::PowerPC | Arch::PowerPC64, _) => false, @@ -405,7 +410,7 @@ fn update_target_reliable_float_cfg(sess: &Session, cfg: &mut TargetConfig) { (Arch::Sparc, _) => false, // Stack alignment bug . NB: tests may // not fail if our compiler-builtins is linked. (fixed in llvm21) - (Arch::X86, _) if lt_21_0_0 => false, + (Arch::X86, _) if major < 21 => false, // MinGW ABI bugs (Arch::X86_64, Os::Windows) if *target_env == Env::Gnu && *target_abi != Abi::Llvm => false, // There are no known problems on other platforms, so the only requirement is that symbols diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 83d7fa0c9153..d8b77369a34f 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -15,7 +15,7 @@ use rustc_target::callconv::{CastTarget, FnAbi}; use crate::abi::{FnAbiLlvmExt, LlvmType}; use crate::common; use crate::context::{CodegenCx, GenericCx, SCx}; -use crate::llvm::{self, FALSE, Metadata, TRUE, ToLlvmBool, Type, Value}; +use crate::llvm::{self, FALSE, Metadata, TRUE, ToGeneric, ToLlvmBool, Type, Value}; use crate::type_of::LayoutLlvmExt; impl PartialEq for Type { diff --git a/compiler/rustc_codegen_llvm/src/typetree.rs b/compiler/rustc_codegen_llvm/src/typetree.rs index 513a832e7fe8..4f433f273c8c 100644 --- a/compiler/rustc_codegen_llvm/src/typetree.rs +++ b/compiler/rustc_codegen_llvm/src/typetree.rs @@ -1,15 +1,10 @@ -use rustc_ast::expand::typetree::FncTree; -#[cfg(feature = "llvm_enzyme")] -use { - crate::attributes, - crate::llvm::EnzymeWrapper, - rustc_ast::expand::typetree::TypeTree as RustTypeTree, - std::ffi::{CString, c_char, c_uint}, -}; +use std::ffi::{CString, c_char, c_uint}; -use crate::llvm::{self, Value}; +use rustc_ast::expand::typetree::{FncTree, TypeTree as RustTypeTree}; + +use crate::attributes; +use crate::llvm::{self, EnzymeWrapper, Value}; -#[cfg(feature = "llvm_enzyme")] fn to_enzyme_typetree( rust_typetree: RustTypeTree, _data_layout: &str, @@ -19,7 +14,6 @@ fn to_enzyme_typetree( process_typetree_recursive(&mut enzyme_tt, &rust_typetree, &[], llcx); enzyme_tt } -#[cfg(feature = "llvm_enzyme")] fn process_typetree_recursive( enzyme_tt: &mut llvm::TypeTree, rust_typetree: &RustTypeTree, @@ -57,13 +51,21 @@ fn process_typetree_recursive( } } -#[cfg(feature = "llvm_enzyme")] +#[cfg_attr(not(feature = "llvm_enzyme"), allow(unused))] pub(crate) fn add_tt<'ll>( llmod: &'ll llvm::Module, llcx: &'ll llvm::Context, fn_def: &'ll Value, tt: FncTree, ) { + // TypeTree processing uses functions from Enzyme, which we might not have available if we did + // not build this compiler with `llvm_enzyme`. This feature is not strictly necessary, but + // skipping this function increases the chance that Enzyme fails to compile some code. + // FIXME(autodiff): In the future we should conditionally run this function even without the + // `llvm_enzyme` feature, in case that libEnzyme was provided via rustup. + #[cfg(not(feature = "llvm_enzyme"))] + return; + let inputs = tt.args; let ret_tt: RustTypeTree = tt.ret; @@ -113,13 +115,3 @@ pub(crate) fn add_tt<'ll>( enzyme_wrapper.tree_to_string_free(c_str.as_ptr()); } } - -#[cfg(not(feature = "llvm_enzyme"))] -pub(crate) fn add_tt<'ll>( - _llmod: &'ll llvm::Module, - _llcx: &'ll llvm::Context, - _fn_def: &'ll Value, - _tt: FncTree, -) { - unimplemented!() -} diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs index 8d77c25bbf90..b23415a732cc 100644 --- a/compiler/rustc_codegen_llvm/src/va_arg.rs +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -7,7 +7,7 @@ use rustc_codegen_ssa::traits::{ }; use rustc_middle::ty::Ty; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; -use rustc_target::spec::{Abi, Arch}; +use rustc_target::spec::{Abi, Arch, Env}; use crate::builder::Builder; use crate::llvm::{Type, Value}; @@ -782,6 +782,129 @@ fn x86_64_sysv64_va_arg_from_memory<'ll, 'tcx>( mem_addr } +fn emit_hexagon_va_arg_musl<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, +) -> &'ll Value { + // Implementation of va_arg for Hexagon musl target. + // Based on LLVM's HexagonBuiltinVaList implementation. + // + // struct __va_list_tag { + // void *__current_saved_reg_area_pointer; + // void *__saved_reg_area_end_pointer; + // void *__overflow_area_pointer; + // }; + // + // All variadic arguments are passed on the stack, but the musl implementation + // uses a register save area for compatibility. + let va_list_addr = list.immediate(); + let layout = bx.cx.layout_of(target_ty); + let ptr_align_abi = bx.tcx().data_layout.pointer_align().abi; + let ptr_size = bx.tcx().data_layout.pointer_size().bytes(); + + // Check if argument fits in register save area + let maybe_reg = bx.append_sibling_block("va_arg.maybe_reg"); + let from_overflow = bx.append_sibling_block("va_arg.from_overflow"); + let end = bx.append_sibling_block("va_arg.end"); + + // Load the three pointers from va_list + let current_ptr_addr = va_list_addr; + let end_ptr_addr = bx.inbounds_ptradd(va_list_addr, bx.const_usize(ptr_size)); + let overflow_ptr_addr = bx.inbounds_ptradd(va_list_addr, bx.const_usize(2 * ptr_size)); + + let current_ptr = bx.load(bx.type_ptr(), current_ptr_addr, ptr_align_abi); + let end_ptr = bx.load(bx.type_ptr(), end_ptr_addr, ptr_align_abi); + let overflow_ptr = bx.load(bx.type_ptr(), overflow_ptr_addr, ptr_align_abi); + + // Align current pointer based on argument type size (following LLVM's implementation) + // Arguments <= 32 bits (4 bytes) use 4-byte alignment, > 32 bits use 8-byte alignment + let type_size_bits = bx.cx.size_of(target_ty).bits(); + let arg_align = if type_size_bits > 32 { + Align::from_bytes(8).unwrap() + } else { + Align::from_bytes(4).unwrap() + }; + let aligned_current = round_pointer_up_to_alignment(bx, current_ptr, arg_align, bx.type_ptr()); + + // Calculate next pointer position (following LLVM's logic) + // Arguments <= 32 bits take 4 bytes, > 32 bits take 8 bytes + let arg_size = if type_size_bits > 32 { 8 } else { 4 }; + let next_ptr = bx.inbounds_ptradd(aligned_current, bx.const_usize(arg_size)); + + // Check if argument fits in register save area + let fits_in_regs = bx.icmp(IntPredicate::IntULE, next_ptr, end_ptr); + bx.cond_br(fits_in_regs, maybe_reg, from_overflow); + + // Load from register save area + bx.switch_to_block(maybe_reg); + let reg_value_addr = aligned_current; + // Update current pointer + bx.store(next_ptr, current_ptr_addr, ptr_align_abi); + bx.br(end); + + // Load from overflow area (stack) + bx.switch_to_block(from_overflow); + + // Align overflow pointer using the same alignment rules + let aligned_overflow = + round_pointer_up_to_alignment(bx, overflow_ptr, arg_align, bx.type_ptr()); + + let overflow_value_addr = aligned_overflow; + // Update overflow pointer - use the same size calculation + let next_overflow = bx.inbounds_ptradd(aligned_overflow, bx.const_usize(arg_size)); + bx.store(next_overflow, overflow_ptr_addr, ptr_align_abi); + + // IMPORTANT: Also update the current saved register area pointer to match + // This synchronizes the pointers when switching to overflow area + bx.store(next_overflow, current_ptr_addr, ptr_align_abi); + bx.br(end); + + // Return the value + bx.switch_to_block(end); + let value_addr = + bx.phi(bx.type_ptr(), &[reg_value_addr, overflow_value_addr], &[maybe_reg, from_overflow]); + bx.load(layout.llvm_type(bx), value_addr, layout.align.abi) +} + +fn emit_hexagon_va_arg_bare_metal<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, +) -> &'ll Value { + // Implementation of va_arg for Hexagon bare-metal (non-musl) targets. + // Based on LLVM's EmitVAArgForHexagon implementation. + // + // va_list is a simple pointer (char *) + let va_list_addr = list.immediate(); + let layout = bx.cx.layout_of(target_ty); + let ptr_align_abi = bx.tcx().data_layout.pointer_align().abi; + + // Load current pointer from va_list + let current_ptr = bx.load(bx.type_ptr(), va_list_addr, ptr_align_abi); + + // Handle address alignment for types with alignment > 4 bytes + let ty_align = layout.align.abi; + let aligned_ptr = if ty_align.bytes() > 4 { + // Ensure alignment is a power of 2 + debug_assert!(ty_align.bytes().is_power_of_two(), "Alignment is not power of 2!"); + round_pointer_up_to_alignment(bx, current_ptr, ty_align, bx.type_ptr()) + } else { + current_ptr + }; + + // Calculate offset: round up type size to 4-byte boundary (minimum stack slot size) + let type_size = layout.size.bytes(); + let offset = type_size.next_multiple_of(4); // align to 4 bytes + + // Update va_list to point to next argument + let next_ptr = bx.inbounds_ptradd(aligned_ptr, bx.const_usize(offset)); + bx.store(next_ptr, va_list_addr, ptr_align_abi); + + // Load and return the argument value + bx.load(layout.llvm_type(bx), aligned_ptr, layout.align.abi) +} + fn emit_xtensa_va_arg<'ll, 'tcx>( bx: &mut Builder<'_, 'll, 'tcx>, list: OperandRef<'tcx, &'ll Value>, @@ -966,6 +1089,13 @@ pub(super) fn emit_va_arg<'ll, 'tcx>( // This includes `target.is_like_darwin`, which on x86_64 targets is like sysv64. Arch::X86_64 => emit_x86_64_sysv64_va_arg(bx, addr, target_ty), Arch::Xtensa => emit_xtensa_va_arg(bx, addr, target_ty), + Arch::Hexagon => { + if target.env == Env::Musl { + emit_hexagon_va_arg_musl(bx, addr, target_ty) + } else { + emit_hexagon_va_arg_bare_metal(bx, addr, target_ty) + } + } // For all other architecture/OS combinations fall back to using // the LLVM va_arg instruction. // https://llvm.org/docs/LangRef.html#va-arg-instruction diff --git a/compiler/rustc_codegen_ssa/src/back/link.rs b/compiler/rustc_codegen_ssa/src/back/link.rs index 9b789d9e62b7..e4e954d22611 100644 --- a/compiler/rustc_codegen_ssa/src/back/link.rs +++ b/compiler/rustc_codegen_ssa/src/back/link.rs @@ -103,17 +103,18 @@ pub fn link_binary( }); if outputs.outputs.should_link() { - let tmpdir = TempDirBuilder::new() - .prefix("rustc") - .tempdir() - .unwrap_or_else(|error| sess.dcx().emit_fatal(errors::CreateTempDir { error })); - let path = MaybeTempDir::new(tmpdir, sess.opts.cg.save_temps); let output = out_filename( sess, crate_type, outputs, codegen_results.crate_info.local_crate_name, ); + let tmpdir = TempDirBuilder::new() + .prefix("rustc") + .tempdir_in(output.parent().unwrap_or_else(|| Path::new("."))) + .unwrap_or_else(|error| sess.dcx().emit_fatal(errors::CreateTempDir { error })); + let path = MaybeTempDir::new(tmpdir, sess.opts.cg.save_temps); + let crate_name = format!("{}", codegen_results.crate_info.local_crate_name); let out_filename = output.file_for_writing( outputs, diff --git a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs index 473598bad394..8135fd43dd93 100644 --- a/compiler/rustc_codegen_ssa/src/codegen_attrs.rs +++ b/compiler/rustc_codegen_ssa/src/codegen_attrs.rs @@ -350,6 +350,9 @@ fn process_builtin_attrs( codegen_fn_attrs.flags |= CodegenFnAttrFlags::EXTERNALLY_IMPLEMENTABLE_ITEM; } } + AttributeKind::ThreadLocal => { + codegen_fn_attrs.flags |= CodegenFnAttrFlags::THREAD_LOCAL + } _ => {} } } @@ -366,7 +369,6 @@ fn process_builtin_attrs( sym::rustc_allocator_zeroed => { codegen_fn_attrs.flags |= CodegenFnAttrFlags::ALLOCATOR_ZEROED } - sym::thread_local => codegen_fn_attrs.flags |= CodegenFnAttrFlags::THREAD_LOCAL, sym::instruction_set => { codegen_fn_attrs.instruction_set = parse_instruction_set_attr(tcx, attr) } diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index cce33107e2c2..d22546dee565 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -9,12 +9,12 @@ use rustc_lint_defs::builtin::TAIL_CALL_TRACK_CALLER; use rustc_middle::mir::{self, AssertKind, InlineAsmMacro, SwitchTargets, UnwindTerminateReason}; use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, ValidityRequirement}; use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; -use rustc_middle::ty::{self, Instance, Ty}; +use rustc_middle::ty::{self, Instance, Ty, TypeVisitableExt}; use rustc_middle::{bug, span_bug}; use rustc_session::config::OptLevel; use rustc_span::Span; use rustc_span::source_map::Spanned; -use rustc_target::callconv::{ArgAbi, CastTarget, FnAbi, PassMode}; +use rustc_target::callconv::{ArgAbi, ArgAttributes, CastTarget, FnAbi, PassMode}; use tracing::{debug, info}; use super::operand::OperandRef; @@ -1036,6 +1036,59 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { _ => bug!("{} is not callable", callee.layout.ty), }; + if let Some(instance) = instance + && let Some(name) = bx.tcx().codegen_fn_attrs(instance.def_id()).symbol_name + && name.as_str().starts_with("llvm.") + // This is the only LLVM intrinsic we use that unwinds + // FIXME either add unwind support to codegen_llvm_intrinsic_call or replace usage of + // this intrinsic with something else + && name.as_str() != "llvm.wasm.throw" + { + assert!(!instance.args.has_infer()); + assert!(!instance.args.has_escaping_bound_vars()); + + let result_layout = + self.cx.layout_of(self.monomorphized_place_ty(destination.as_ref())); + + let return_dest = if result_layout.is_zst() { + ReturnDest::Nothing + } else if let Some(index) = destination.as_local() { + match self.locals[index] { + LocalRef::Place(dest) => ReturnDest::Store(dest), + LocalRef::UnsizedPlace(_) => bug!("return type must be sized"), + LocalRef::PendingOperand => { + // Handle temporary places, specifically `Operand` ones, as + // they don't have `alloca`s. + ReturnDest::DirectOperand(index) + } + LocalRef::Operand(_) => bug!("place local already assigned to"), + } + } else { + ReturnDest::Store(self.codegen_place(bx, destination.as_ref())) + }; + + let args = + args.into_iter().map(|arg| self.codegen_operand(bx, &arg.node)).collect::>(); + + self.set_debug_loc(bx, source_info); + + let llret = + bx.codegen_llvm_intrinsic_call(instance, &args, self.mir[helper.bb].is_cleanup); + + if let Some(target) = target { + self.store_return( + bx, + return_dest, + &ArgAbi { layout: result_layout, mode: PassMode::Direct(ArgAttributes::new()) }, + llret, + ); + return helper.funclet_br(self, bx, target, mergeable_succ); + } else { + bx.unreachable(); + return MergingSucc::False; + } + } + // FIXME(eddyb) avoid computing this if possible, when `instance` is // available - right now `sig` is only needed for getting the `abi` // and figuring out how many extra args were passed to a C-variadic `fn`. diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs index 11b6ab3cdf1a..abdac4c7c372 100644 --- a/compiler/rustc_codegen_ssa/src/mir/constant.rs +++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs @@ -77,22 +77,21 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { .flatten() .map(|val| { // A SIMD type has a single field, which is an array. - let fields = val.unwrap_branch(); + let fields = val.to_branch(); assert_eq!(fields.len(), 1); - let array = fields[0].unwrap_branch(); + let array = fields[0].to_branch(); // Iterate over the array elements to obtain the values in the vector. let values: Vec<_> = array .iter() .map(|field| { - if let Some(prim) = field.try_to_scalar() { - let layout = bx.layout_of(field_ty); - let BackendRepr::Scalar(scalar) = layout.backend_repr else { - bug!("from_const: invalid ByVal layout: {:#?}", layout); - }; - bx.scalar_to_backend(prim, scalar, bx.immediate_backend_type(layout)) - } else { + let Some(prim) = field.try_to_scalar() else { bug!("field is not a scalar {:?}", field) - } + }; + let layout = bx.layout_of(field_ty); + let BackendRepr::Scalar(scalar) = layout.backend_repr else { + bug!("from_const: invalid ByVal layout: {:#?}", layout); + }; + bx.scalar_to_backend(prim, scalar, bx.immediate_backend_type(layout)) }) .collect(); bx.const_vector(&values) diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs index aeb740118234..f4fae40d8828 100644 --- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs @@ -102,7 +102,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { }; let parse_atomic_ordering = |ord: ty::Value<'tcx>| { - let discr = ord.valtree.unwrap_branch()[0].unwrap_leaf(); + let discr = ord.to_branch()[0].to_leaf(); discr.to_atomic_ordering() }; diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs index 6fd118a60d4d..78dfecdd1818 100644 --- a/compiler/rustc_codegen_ssa/src/mir/operand.rs +++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs @@ -1056,6 +1056,17 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { OperandRef { move_annotation, ..self.codegen_consume(bx, place.as_ref()) } } + mir::Operand::RuntimeChecks(checks) => { + let layout = bx.layout_of(bx.tcx().types.bool); + let BackendRepr::Scalar(scalar) = layout.backend_repr else { + bug!("from_const: invalid ByVal layout: {:#?}", layout); + }; + let x = Scalar::from_bool(checks.value(bx.tcx().sess)); + let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout)); + let val = OperandValue::Immediate(llval); + OperandRef { val, layout, move_annotation: None } + } + mir::Operand::Constant(ref constant) => { let constant_ty = self.monomorphize(constant.ty()); // Most SIMD vector constants should be passed as immediates. diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs index de626d04e785..ca8c8dd06ba6 100644 --- a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs +++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs @@ -619,21 +619,6 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } } - mir::Rvalue::NullaryOp(ref null_op) => { - let val = match null_op { - mir::NullOp::RuntimeChecks(kind) => { - let val = kind.value(bx.tcx().sess); - bx.cx().const_bool(val) - } - }; - let tcx = self.cx.tcx(); - OperandRef { - val: OperandValue::Immediate(val), - layout: self.cx.layout_of(null_op.ty(tcx)), - move_annotation: None, - } - } - mir::Rvalue::ThreadLocalRef(def_id) => { assert!(bx.cx().tcx().is_static(def_id)); let layout = bx.layout_of(bx.cx().tcx().static_ptr_ty(def_id, bx.typing_env())); diff --git a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs index 187e4b90656a..04183c2801e7 100644 --- a/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs +++ b/compiler/rustc_codegen_ssa/src/traits/intrinsic.rs @@ -25,6 +25,13 @@ pub trait IntrinsicCallBuilderMethods<'tcx>: BackendTypes { span: Span, ) -> Result<(), ty::Instance<'tcx>>; + fn codegen_llvm_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OperandRef<'tcx, Self::Value>], + is_cleanup: bool, + ) -> Self::Value; + fn abort(&mut self); fn assume(&mut self, val: Self::Value); fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value; diff --git a/compiler/rustc_const_eval/src/check_consts/check.rs b/compiler/rustc_const_eval/src/check_consts/check.rs index 78e4066ca910..b06b407a6085 100644 --- a/compiler/rustc_const_eval/src/check_consts/check.rs +++ b/compiler/rustc_const_eval/src/check_consts/check.rs @@ -251,7 +251,7 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> { let mut transient = DenseBitSet::new_filled(ccx.body.local_decls.len()); // Make sure to only visit reachable blocks, the dataflow engine can ICE otherwise. for (bb, data) in traversal::reachable(&ccx.body) { - if matches!(data.terminator().kind, TerminatorKind::Return) { + if data.terminator().kind == TerminatorKind::Return { let location = ccx.body.terminator_loc(bb); maybe_storage_live.seek_after_primary_effect(location); // If a local may be live here, it is definitely not transient. @@ -645,7 +645,6 @@ impl<'tcx> Visitor<'tcx> for Checker<'_, 'tcx> { Rvalue::Cast(_, _, _) => {} - Rvalue::NullaryOp(NullOp::RuntimeChecks(_)) => {} Rvalue::ShallowInitBox(_, _) => {} Rvalue::UnaryOp(op, operand) => { diff --git a/compiler/rustc_const_eval/src/check_consts/qualifs.rs b/compiler/rustc_const_eval/src/check_consts/qualifs.rs index f50c6af53bf1..113e0d66c48a 100644 --- a/compiler/rustc_const_eval/src/check_consts/qualifs.rs +++ b/compiler/rustc_const_eval/src/check_consts/qualifs.rs @@ -230,9 +230,7 @@ where F: FnMut(Local) -> bool, { match rvalue { - Rvalue::ThreadLocalRef(_) | Rvalue::NullaryOp(..) => { - Q::in_any_value_of_ty(cx, rvalue.ty(cx.body, cx.tcx)) - } + Rvalue::ThreadLocalRef(_) => Q::in_any_value_of_ty(cx, rvalue.ty(cx.body, cx.tcx)), Rvalue::Discriminant(place) => in_place::(cx, in_local, place.as_ref()), @@ -314,7 +312,7 @@ where // i.e., we treat all qualifs as non-structural for deref projections. Generally, // we can say very little about `*ptr` even if we know that `ptr` satisfies all // sorts of properties. - if matches!(elem, ProjectionElem::Deref) { + if elem == ProjectionElem::Deref { // We have to assume that this qualifies. return true; } @@ -340,6 +338,7 @@ where Operand::Copy(place) | Operand::Move(place) => { return in_place::(cx, in_local, place.as_ref()); } + Operand::RuntimeChecks(_) => return Q::in_any_value_of_ty(cx, cx.tcx.types.bool), Operand::Constant(c) => c, }; diff --git a/compiler/rustc_const_eval/src/check_consts/resolver.rs b/compiler/rustc_const_eval/src/check_consts/resolver.rs index e6e3948305af..d4cc21996aea 100644 --- a/compiler/rustc_const_eval/src/check_consts/resolver.rs +++ b/compiler/rustc_const_eval/src/check_consts/resolver.rs @@ -198,7 +198,6 @@ where | mir::Rvalue::ThreadLocalRef(..) | mir::Rvalue::Repeat(..) | mir::Rvalue::BinaryOp(..) - | mir::Rvalue::NullaryOp(..) | mir::Rvalue::UnaryOp(..) | mir::Rvalue::Discriminant(..) | mir::Rvalue::Aggregate(..) diff --git a/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs b/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs index 438aed41b8be..f41b68bb1f7e 100644 --- a/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/dummy_machine.rs @@ -122,6 +122,13 @@ impl<'tcx> interpret::Machine<'tcx> for DummyMachine { unimplemented!() } + #[inline(always)] + fn runtime_checks(_ecx: &InterpCx<'tcx, Self>, r: RuntimeChecks) -> InterpResult<'tcx, bool> { + // Runtime checks have different value depending on the crate they are codegenned in. + // Verify we aren't trying to evaluate them in mir-optimizations. + panic!("compiletime machine evaluated {r:?}") + } + fn binary_ptr_op( ecx: &InterpCx<'tcx, Self>, bin_op: BinOp, diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index 0c677b34df7b..7538130e9d92 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -637,6 +637,16 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> { Err(ConstEvalErrKind::AssertFailure(err)).into() } + #[inline(always)] + fn runtime_checks( + _ecx: &InterpCx<'tcx, Self>, + _r: mir::RuntimeChecks, + ) -> InterpResult<'tcx, bool> { + // We can't look at `tcx.sess` here as that can differ across crates, which can lead to + // unsound differences in evaluating the same constant at different instantiation sites. + interp_ok(true) + } + fn binary_ptr_op( _ecx: &InterpCx<'tcx, Self>, _bin_op: mir::BinOp, diff --git a/compiler/rustc_const_eval/src/const_eval/valtrees.rs b/compiler/rustc_const_eval/src/const_eval/valtrees.rs index 7c41258ebfe5..b771addb8df5 100644 --- a/compiler/rustc_const_eval/src/const_eval/valtrees.rs +++ b/compiler/rustc_const_eval/src/const_eval/valtrees.rs @@ -36,13 +36,17 @@ fn branches<'tcx>( // For enums, we prepend their variant index before the variant's fields so we can figure out // the variant again when just seeing a valtree. if let Some(variant) = variant { - branches.push(ty::ValTree::from_scalar_int(*ecx.tcx, variant.as_u32().into())); + branches.push(ty::Const::new_value( + *ecx.tcx, + ty::ValTree::from_scalar_int(*ecx.tcx, variant.as_u32().into()), + ecx.tcx.types.u32, + )); } for i in 0..field_count { let field = ecx.project_field(&place, FieldIdx::from_usize(i)).unwrap(); let valtree = const_to_valtree_inner(ecx, &field, num_nodes)?; - branches.push(valtree); + branches.push(ty::Const::new_value(*ecx.tcx, valtree, field.layout.ty)); } // Have to account for ZSTs here @@ -65,7 +69,7 @@ fn slice_branches<'tcx>( for i in 0..n { let place_elem = ecx.project_index(place, i).unwrap(); let valtree = const_to_valtree_inner(ecx, &place_elem, num_nodes)?; - elems.push(valtree); + elems.push(ty::Const::new_value(*ecx.tcx, valtree, place_elem.layout.ty)); } Ok(ty::ValTree::from_branches(*ecx.tcx, elems)) @@ -200,8 +204,8 @@ fn reconstruct_place_meta<'tcx>( &ObligationCause::dummy(), |ty| ty, || { - let branches = last_valtree.unwrap_branch(); - last_valtree = *branches.last().unwrap(); + let branches = last_valtree.to_branch(); + last_valtree = branches.last().unwrap().to_value().valtree; debug!(?branches, ?last_valtree); }, ); @@ -212,7 +216,7 @@ fn reconstruct_place_meta<'tcx>( }; // Get the number of elements in the unsized field. - let num_elems = last_valtree.unwrap_branch().len(); + let num_elems = last_valtree.to_branch().len(); MemPlaceMeta::Meta(Scalar::from_target_usize(num_elems as u64, &tcx)) } @@ -274,7 +278,7 @@ pub fn valtree_to_const_value<'tcx>( mir::ConstValue::ZeroSized } ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char | ty::RawPtr(_, _) => { - mir::ConstValue::Scalar(Scalar::Int(cv.valtree.unwrap_leaf())) + mir::ConstValue::Scalar(Scalar::Int(cv.to_leaf())) } ty::Pat(ty, _) => { let cv = ty::Value { valtree: cv.valtree, ty }; @@ -301,12 +305,13 @@ pub fn valtree_to_const_value<'tcx>( || matches!(cv.ty.kind(), ty::Adt(def, _) if def.is_struct())) { // A Scalar tuple/struct; we can avoid creating an allocation. - let branches = cv.valtree.unwrap_branch(); + let branches = cv.to_branch(); // Find the non-ZST field. (There can be aligned ZST!) for (i, &inner_valtree) in branches.iter().enumerate() { let field = layout.field(&LayoutCx::new(tcx, typing_env), i); if !field.is_zst() { - let cv = ty::Value { valtree: inner_valtree, ty: field.ty }; + let cv = + ty::Value { valtree: inner_valtree.to_value().valtree, ty: field.ty }; return valtree_to_const_value(tcx, typing_env, cv); } } @@ -381,7 +386,7 @@ fn valtree_into_mplace<'tcx>( // Zero-sized type, nothing to do. } ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char | ty::RawPtr(..) => { - let scalar_int = valtree.unwrap_leaf(); + let scalar_int = valtree.to_leaf(); debug!("writing trivial valtree {:?} to place {:?}", scalar_int, place); ecx.write_immediate(Immediate::Scalar(scalar_int.into()), place).unwrap(); } @@ -391,13 +396,13 @@ fn valtree_into_mplace<'tcx>( ecx.write_immediate(imm, place).unwrap(); } ty::Adt(_, _) | ty::Tuple(_) | ty::Array(_, _) | ty::Str | ty::Slice(_) => { - let branches = valtree.unwrap_branch(); + let branches = valtree.to_branch(); // Need to downcast place for enums let (place_adjusted, branches, variant_idx) = match ty.kind() { ty::Adt(def, _) if def.is_enum() => { // First element of valtree corresponds to variant - let scalar_int = branches[0].unwrap_leaf(); + let scalar_int = branches[0].to_leaf(); let variant_idx = VariantIdx::from_u32(scalar_int.to_u32()); let variant = def.variant(variant_idx); debug!(?variant); @@ -425,7 +430,7 @@ fn valtree_into_mplace<'tcx>( }; debug!(?place_inner); - valtree_into_mplace(ecx, &place_inner, *inner_valtree); + valtree_into_mplace(ecx, &place_inner, inner_valtree.to_value().valtree); dump_place(ecx, &place_inner); } diff --git a/compiler/rustc_const_eval/src/interpret/call.rs b/compiler/rustc_const_eval/src/interpret/call.rs index 312ebe7ddd09..94c6fd1b3238 100644 --- a/compiler/rustc_const_eval/src/interpret/call.rs +++ b/compiler/rustc_const_eval/src/interpret/call.rs @@ -283,7 +283,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { 'tcx: 'y, { assert_eq!(callee_ty, callee_abi.layout.ty); - if matches!(callee_abi.mode, PassMode::Ignore) { + if callee_abi.mode == PassMode::Ignore { // This one is skipped. Still must be made live though! if !already_live { self.storage_live(callee_arg.as_local().unwrap())?; diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 44c817b33184..d70d157d8808 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -861,7 +861,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } } else { // unsigned - if matches!(mir_op, BinOp::Add) { + if mir_op == BinOp::Add { // max unsigned Scalar::from_uint(size.unsigned_int_max(), size) } else { diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs index 20de47683122..33a115384a88 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics/simd.rs @@ -545,7 +545,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { let (right, right_len) = self.project_to_simd(&args[1])?; let (dest, dest_len) = self.project_to_simd(&dest)?; - let index = generic_args[2].expect_const().to_value().valtree.unwrap_branch(); + let index = generic_args[2].expect_const().to_branch(); let index_len = index.len(); assert_eq!(left_len, right_len); @@ -553,7 +553,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { for i in 0..dest_len { let src_index: u64 = - index[usize::try_from(i).unwrap()].unwrap_leaf().to_u32().into(); + index[usize::try_from(i).unwrap()].to_leaf().to_u32().into(); let dest = self.project_index(&dest, i)?; let val = if src_index < left_len { @@ -657,9 +657,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.check_simd_ptr_alignment( ptr, dest_layout, - generic_args[3].expect_const().to_value().valtree.unwrap_branch()[0] - .unwrap_leaf() - .to_simd_alignment(), + generic_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment(), )?; for i in 0..dest_len { @@ -689,9 +687,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.check_simd_ptr_alignment( ptr, args[2].layout, - generic_args[3].expect_const().to_value().valtree.unwrap_branch()[0] - .unwrap_leaf() - .to_simd_alignment(), + generic_args[3].expect_const().to_branch()[0].to_leaf().to_simd_alignment(), )?; for i in 0..vals_len { diff --git a/compiler/rustc_const_eval/src/interpret/machine.rs b/compiler/rustc_const_eval/src/interpret/machine.rs index 753b2dd3b8ea..62ca47d23b4c 100644 --- a/compiler/rustc_const_eval/src/interpret/machine.rs +++ b/compiler/rustc_const_eval/src/interpret/machine.rs @@ -298,7 +298,7 @@ pub trait Machine<'tcx>: Sized { interp_ok(()) } - /// Determines the result of a `NullaryOp::RuntimeChecks` invocation. + /// Determines the result of a `Operand::RuntimeChecks` invocation. fn runtime_checks( _ecx: &InterpCx<'tcx, Self>, r: mir::RuntimeChecks, @@ -680,16 +680,6 @@ pub macro compile_time_machine(<$tcx: lifetime>) { true } - #[inline(always)] - fn runtime_checks( - _ecx: &InterpCx<$tcx, Self>, - _r: mir::RuntimeChecks, - ) -> InterpResult<$tcx, bool> { - // We can't look at `tcx.sess` here as that can differ across crates, which can lead to - // unsound differences in evaluating the same constant at different instantiation sites. - interp_ok(true) - } - #[inline(always)] fn adjust_global_allocation<'b>( _ecx: &InterpCx<$tcx, Self>, diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs index bac3a9da48d9..862fe4779080 100644 --- a/compiler/rustc_const_eval/src/interpret/memory.rs +++ b/compiler/rustc_const_eval/src/interpret/memory.rs @@ -327,7 +327,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { return Err(ConstEvalErrKind::ConstMakeGlobalWithOffset(ptr)).into(); } - if matches!(self.tcx.try_get_global_alloc(alloc_id), Some(_)) { + if self.tcx.try_get_global_alloc(alloc_id).is_some() { // This points to something outside the current interpreter. return Err(ConstEvalErrKind::ConstMakeGlobalPtrIsNonHeap(ptr)).into(); } @@ -981,7 +981,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { msg: CheckInAllocMsg, ) -> InterpResult<'tcx, (Size, Align)> { let info = self.get_alloc_info(id); - if matches!(info.kind, AllocKind::Dead) { + if info.kind == AllocKind::Dead { throw_ub!(PointerUseAfterFree(id, msg)) } interp_ok((info.size, info.align)) @@ -1072,7 +1072,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // Recurse, if there is data here. // Do this *before* invoking the callback, as the callback might mutate the // allocation and e.g. replace all provenance by wildcards! - if matches!(info.kind, AllocKind::LiveData) { + if info.kind == AllocKind::LiveData { let alloc = self.get_alloc_raw(id)?; for prov in alloc.provenance().provenances() { if let Some(id) = prov.get_alloc_id() { @@ -1605,7 +1605,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { match self.ptr_try_get_alloc_id(ptr, 0) { Ok((alloc_id, offset, _)) => { let info = self.get_alloc_info(alloc_id); - if matches!(info.kind, AllocKind::TypeId) { + if info.kind == AllocKind::TypeId { // We *could* actually precisely answer this question since here, // the offset *is* the integer value. But the entire point of making // this a pointer is not to leak the integer value, so we say everything diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs index d3d119c8fc9c..9a956259ba57 100644 --- a/compiler/rustc_const_eval/src/interpret/operand.rs +++ b/compiler/rustc_const_eval/src/interpret/operand.rs @@ -845,6 +845,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { // FIXME: do some more logic on `move` to invalidate the old location &Copy(place) | &Move(place) => self.eval_place_to_op(place, layout)?, + &RuntimeChecks(checks) => { + let val = M::runtime_checks(self, checks)?; + ImmTy::from_bool(val, self.tcx()).into() + } + Constant(constant) => { let c = self.instantiate_from_current_frame_and_normalize_erasing_regions( constant.const_, diff --git a/compiler/rustc_const_eval/src/interpret/operator.rs b/compiler/rustc_const_eval/src/interpret/operator.rs index 8548b774ddb4..ca8c096d3ab4 100644 --- a/compiler/rustc_const_eval/src/interpret/operator.rs +++ b/compiler/rustc_const_eval/src/interpret/operator.rs @@ -1,7 +1,6 @@ use either::Either; use rustc_abi::Size; use rustc_apfloat::{Float, FloatConvert}; -use rustc_middle::mir::NullOp; use rustc_middle::mir::interpret::{InterpResult, PointerArithmetic, Scalar}; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::{self, FloatTy, ScalarInt}; @@ -505,11 +504,4 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { } } } - - pub fn nullary_op(&self, null_op: NullOp) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> { - use rustc_middle::mir::NullOp::*; - interp_ok(match null_op { - RuntimeChecks(r) => ImmTy::from_bool(M::runtime_checks(self, r)?, *self.tcx), - }) - } } diff --git a/compiler/rustc_const_eval/src/interpret/step.rs b/compiler/rustc_const_eval/src/interpret/step.rs index 88a116094758..051c6c344506 100644 --- a/compiler/rustc_const_eval/src/interpret/step.rs +++ b/compiler/rustc_const_eval/src/interpret/step.rs @@ -203,11 +203,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { self.write_immediate(*result, &dest)?; } - NullaryOp(null_op) => { - let val = self.nullary_op(null_op)?; - self.write_immediate(*val, &dest)?; - } - Aggregate(box ref kind, ref operands) => { self.write_aggregate(kind, operands, &dest)?; } @@ -392,7 +387,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { move_definitely_disjoint: bool, ) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> { interp_ok(match op { - mir::Operand::Copy(_) | mir::Operand::Constant(_) => { + mir::Operand::Copy(_) | mir::Operand::Constant(_) | mir::Operand::RuntimeChecks(_) => { // Make a regular copy. let op = self.eval_operand(op, None)?; FnArg::Copy(op) @@ -561,8 +556,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> { if fn_abi.can_unwind { unwind } else { mir::UnwindAction::Unreachable }, )?; // Sanity-check that `eval_fn_call` either pushed a new frame or - // did a jump to another block. - if self.frame_idx() == old_stack && self.frame().loc == old_loc { + // did a jump to another block. We disable the sanity check for functions that + // can't return, since Miri sometimes does have to keep the location the same + // for those (which is fine since execution will continue on a different thread). + if target.is_some() && self.frame_idx() == old_stack && self.frame().loc == old_loc + { span_bug!(terminator.source_info.span, "evaluating this call made no progress"); } } diff --git a/compiler/rustc_const_eval/src/interpret/visitor.rs b/compiler/rustc_const_eval/src/interpret/visitor.rs index b5de10c7dcd1..a8d472bc2ea2 100644 --- a/compiler/rustc_const_eval/src/interpret/visitor.rs +++ b/compiler/rustc_const_eval/src/interpret/visitor.rs @@ -4,7 +4,6 @@ use std::num::NonZero; use rustc_abi::{FieldIdx, FieldsShape, VariantIdx, Variants}; -use rustc_index::IndexVec; use rustc_middle::mir::interpret::InterpResult; use rustc_middle::ty::{self, Ty}; use tracing::trace; @@ -24,18 +23,6 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized { self.ecx().read_discriminant(&v.to_op(self.ecx())?) } - /// This function provides the chance to reorder the order in which fields are visited for - /// `FieldsShape::Aggregate`. - /// - /// The default means we iterate in source declaration order; alternatively this can do some - /// work with `memory_index` to iterate in memory order. - #[inline(always)] - fn aggregate_field_iter( - memory_index: &IndexVec, - ) -> impl Iterator + 'static { - memory_index.indices() - } - // Recursive actions, ready to be overloaded. /// Visits the given value, dispatching as appropriate to more specialized visitors. #[inline(always)] @@ -168,8 +155,8 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized { &FieldsShape::Union(fields) => { self.visit_union(v, fields)?; } - FieldsShape::Arbitrary { memory_index, .. } => { - for idx in Self::aggregate_field_iter(memory_index) { + FieldsShape::Arbitrary { in_memory_order, .. } => { + for idx in in_memory_order.iter().copied() { let field = self.ecx().project_field(v, idx)?; self.visit_field(v, idx.as_usize(), &field)?; } diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs index 63fc9c96f450..7820198f2dcf 100644 --- a/compiler/rustc_driver_impl/src/lib.rs +++ b/compiler/rustc_driver_impl/src/lib.rs @@ -1531,15 +1531,15 @@ fn report_ice( .map(PathBuf::from) .map(|env_var| session_diagnostics::IcePathErrorEnv { env_var }), }); - dcx.emit_note(session_diagnostics::IceVersion { version, triple: tuple }); None } } } else { - dcx.emit_note(session_diagnostics::IceVersion { version, triple: tuple }); None }; + dcx.emit_note(session_diagnostics::IceVersion { version, triple: tuple }); + if let Some((flags, excluded_cargo_defaults)) = rustc_session::utils::extra_compiler_flags() { dcx.emit_note(session_diagnostics::IceFlags { flags: flags.join(" ") }); if excluded_cargo_defaults { diff --git a/compiler/rustc_feature/src/accepted.rs b/compiler/rustc_feature/src/accepted.rs index 7e765918d7b6..13c1f2219bed 100644 --- a/compiler/rustc_feature/src/accepted.rs +++ b/compiler/rustc_feature/src/accepted.rs @@ -61,7 +61,7 @@ declare_features! ( /// Allows explicit discriminants on non-unit enum variants. (accepted, arbitrary_enum_discriminant, "1.66.0", Some(60553)), /// Allows #[cfg(...)] on inline assembly templates and operands. - (accepted, asm_cfg, "CURRENT_RUSTC_VERSION", Some(140364)), + (accepted, asm_cfg, "1.93.0", Some(140364)), /// Allows using `const` operands in inline assembly. (accepted, asm_const, "1.82.0", Some(93332)), /// Allows using `label` operands in inline assembly. @@ -218,7 +218,7 @@ declare_features! ( /// Allows access to crate names passed via `--extern` through prelude. (accepted, extern_prelude, "1.30.0", Some(44660)), /// Allows using `system` as a calling convention with varargs. - (accepted, extern_system_varargs, "CURRENT_RUSTC_VERSION", Some(136946)), + (accepted, extern_system_varargs, "1.93.0", Some(136946)), /// Allows using F16C intrinsics from `core::arch::{x86, x86_64}`. (accepted, f16c_target_feature, "1.68.0", Some(44839)), /// Allows field shorthands (`x` meaning `x: x`) in struct literal expressions. @@ -392,7 +392,7 @@ declare_features! ( /// Allows code like `let x: &'static u32 = &42` to work (RFC 1414). (accepted, rvalue_static_promotion, "1.21.0", Some(38865)), /// Allows use of the `vector` and related s390x target features. - (accepted, s390x_target_feature_vector, "CURRENT_RUSTC_VERSION", Some(145649)), + (accepted, s390x_target_feature_vector, "1.93.0", Some(145649)), /// Allows `Self` in type definitions (RFC 2300). (accepted, self_in_typedefs, "1.32.0", Some(49303)), /// Allows `Self` struct constructor (RFC 2302). diff --git a/compiler/rustc_feature/src/builtin_attrs.rs b/compiler/rustc_feature/src/builtin_attrs.rs index 0b930dc342f2..a7e8515e415f 100644 --- a/compiler/rustc_feature/src/builtin_attrs.rs +++ b/compiler/rustc_feature/src/builtin_attrs.rs @@ -1123,7 +1123,7 @@ pub static BUILTIN_ATTRIBUTES: &[BuiltinAttribute] = &[ template!(Word, List: &[r#""...""#]), DuplicatesOk, EncodeCrossCrate::Yes, ), - rustc_attr!( + rustc_attr!( rustc_offload_kernel, Normal, template!(Word), DuplicatesOk, EncodeCrossCrate::Yes, diff --git a/compiler/rustc_feature/src/removed.rs b/compiler/rustc_feature/src/removed.rs index cd61facb39bb..e5d66364c2a6 100644 --- a/compiler/rustc_feature/src/removed.rs +++ b/compiler/rustc_feature/src/removed.rs @@ -190,7 +190,7 @@ declare_features! ( /// Allows use of unary negate on unsigned integers, e.g., -e for e: u8 (removed, negate_unsigned, "1.0.0", Some(29645), None), /// Allows diverging expressions to fall back to `!` rather than `()`. - (removed, never_type_fallback, "CURRENT_RUSTC_VERSION", Some(65992), Some("removed in favor of unconditional fallback"), 148871), + (removed, never_type_fallback, "1.93.0", Some(65992), Some("removed in favor of unconditional fallback"), 148871), /// Allows `#[no_coverage]` on functions. /// The feature was renamed to `coverage_attribute` and the attribute to `#[coverage(on|off)]` (removed, no_coverage, "1.74.0", Some(84605), Some("renamed to `coverage_attribute`"), 114656), diff --git a/compiler/rustc_feature/src/unstable.rs b/compiler/rustc_feature/src/unstable.rs index fe053935f9e6..692cba8035c4 100644 --- a/compiler/rustc_feature/src/unstable.rs +++ b/compiler/rustc_feature/src/unstable.rs @@ -221,7 +221,7 @@ declare_features! ( (internal, compiler_builtins, "1.13.0", None), /// Allows writing custom MIR (internal, custom_mir, "1.65.0", None), - /// Implementation details of externally implementatble items + /// Implementation details of externally implementable items (internal, eii_internals, "CURRENT_RUSTC_VERSION", None), /// Outputs useful `assert!` messages (unstable, generic_assert, "1.63.0", None), @@ -412,7 +412,7 @@ declare_features! ( (unstable, c_variadic, "1.34.0", Some(44930)), /// Allows defining c-variadic naked functions with any extern ABI that is allowed /// on c-variadic foreign functions. - (unstable, c_variadic_naked_functions, "CURRENT_RUSTC_VERSION", Some(148767)), + (unstable, c_variadic_naked_functions, "1.93.0", Some(148767)), /// Allows the use of `#[cfg(contract_checks)` to check if contract checks are enabled. (unstable, cfg_contract_checks, "1.86.0", Some(128044)), /// Allows the use of `#[cfg(overflow_checks)` to check if integer overflow behaviour. @@ -486,7 +486,7 @@ declare_features! ( /// Allows deriving the From trait on single-field structs. (unstable, derive_from, "1.91.0", Some(144889)), /// Allows giving non-const impls custom diagnostic messages if attempted to be used as const - (unstable, diagnostic_on_const, "CURRENT_RUSTC_VERSION", Some(143874)), + (unstable, diagnostic_on_const, "1.93.0", Some(143874)), /// Allows `#[doc(cfg(...))]`. (unstable, doc_cfg, "1.21.0", Some(43781)), /// Allows `#[doc(masked)]`. @@ -503,7 +503,7 @@ declare_features! ( (incomplete, explicit_tail_calls, "1.72.0", Some(112788)), /// Allows using `#[export_stable]` which indicates that an item is exportable. (incomplete, export_stable, "1.88.0", Some(139939)), - /// Externally implementatble items + /// Externally implementable items (unstable, extern_item_impls, "CURRENT_RUSTC_VERSION", Some(125418)), /// Allows defining `extern type`s. (unstable, extern_types, "1.23.0", Some(43467)), diff --git a/compiler/rustc_fs_util/src/lib.rs b/compiler/rustc_fs_util/src/lib.rs index 7a883a13b72d..e21da4cdddf7 100644 --- a/compiler/rustc_fs_util/src/lib.rs +++ b/compiler/rustc_fs_util/src/lib.rs @@ -1,6 +1,6 @@ use std::ffi::{CString, OsStr}; use std::path::{Path, PathBuf, absolute}; -use std::{env, fs, io}; +use std::{fs, io}; use tempfile::TempDir; @@ -139,8 +139,4 @@ impl<'a, 'b> TempDirBuilder<'a, 'b> { } self.builder.tempdir_in(dir) } - - pub fn tempdir(&self) -> io::Result { - self.tempdir_in(env::temp_dir()) - } } diff --git a/compiler/rustc_graphviz/src/lib.rs b/compiler/rustc_graphviz/src/lib.rs index 9f75578aa636..cd1e573ea28d 100644 --- a/compiler/rustc_graphviz/src/lib.rs +++ b/compiler/rustc_graphviz/src/lib.rs @@ -416,7 +416,7 @@ impl<'a> Id<'a> { /// it in the generated .dot file. They can also provide more /// elaborate (and non-unique) label text that is used in the graphviz /// rendered output. - +/// /// The graph instance is responsible for providing the DOT compatible /// identifiers for the nodes and (optionally) rendered labels for the nodes and /// edges, as well as an identifier for the graph itself. diff --git a/compiler/rustc_hir/src/attrs/data_structures.rs b/compiler/rustc_hir/src/attrs/data_structures.rs index 5f38eaf4d5f4..6d89825b07d6 100644 --- a/compiler/rustc_hir/src/attrs/data_structures.rs +++ b/compiler/rustc_hir/src/attrs/data_structures.rs @@ -530,7 +530,7 @@ pub struct CfgHideShow { pub values: ThinVec, } -#[derive(Clone, Debug, Default, HashStable_Generic, Encodable, Decodable, PrintAttribute)] +#[derive(Clone, Debug, Default, HashStable_Generic, Decodable, PrintAttribute)] pub struct DocAttribute { pub aliases: FxIndexMap, pub hidden: Option, @@ -566,6 +566,62 @@ pub struct DocAttribute { pub no_crate_inject: Option, } +impl rustc_serialize::Encodable for DocAttribute { + fn encode(&self, encoder: &mut E) { + let DocAttribute { + aliases, + hidden, + inline, + cfg, + auto_cfg, + auto_cfg_change, + fake_variadic, + keyword, + attribute, + masked, + notable_trait, + search_unbox, + html_favicon_url, + html_logo_url, + html_playground_url, + html_root_url, + html_no_source, + issue_tracker_base_url, + rust_logo, + test_attrs, + no_crate_inject, + } = self; + rustc_serialize::Encodable::::encode(aliases, encoder); + rustc_serialize::Encodable::::encode(hidden, encoder); + + // FIXME: The `doc(inline)` attribute is never encoded, but is it actually the right thing + // to do? I suspect the condition was broken, should maybe instead not encode anything if we + // have `doc(no_inline)`. + let inline: ThinVec<_> = + inline.iter().filter(|(i, _)| *i != DocInline::Inline).cloned().collect(); + rustc_serialize::Encodable::::encode(&inline, encoder); + + rustc_serialize::Encodable::::encode(cfg, encoder); + rustc_serialize::Encodable::::encode(auto_cfg, encoder); + rustc_serialize::Encodable::::encode(auto_cfg_change, encoder); + rustc_serialize::Encodable::::encode(fake_variadic, encoder); + rustc_serialize::Encodable::::encode(keyword, encoder); + rustc_serialize::Encodable::::encode(attribute, encoder); + rustc_serialize::Encodable::::encode(masked, encoder); + rustc_serialize::Encodable::::encode(notable_trait, encoder); + rustc_serialize::Encodable::::encode(search_unbox, encoder); + rustc_serialize::Encodable::::encode(html_favicon_url, encoder); + rustc_serialize::Encodable::::encode(html_logo_url, encoder); + rustc_serialize::Encodable::::encode(html_playground_url, encoder); + rustc_serialize::Encodable::::encode(html_root_url, encoder); + rustc_serialize::Encodable::::encode(html_no_source, encoder); + rustc_serialize::Encodable::::encode(issue_tracker_base_url, encoder); + rustc_serialize::Encodable::::encode(rust_logo, encoder); + rustc_serialize::Encodable::::encode(test_attrs, encoder); + rustc_serialize::Encodable::::encode(no_crate_inject, encoder); + } +} + /// Represents parsed *built-in* inert attributes. /// /// ## Overview @@ -647,6 +703,9 @@ pub enum AttributeKind { span: Span, }, + /// Represents `#[cfi_encoding]` + CfiEncoding { encoding: Symbol }, + /// Represents `#[rustc_coinductive]`. Coinductive(Span), @@ -803,6 +862,9 @@ pub enum AttributeKind { /// Represents `#[no_implicit_prelude]` NoImplicitPrelude(Span), + /// Represents `#[no_link]` + NoLink, + /// Represents `#[no_mangle]` NoMangle(Span), @@ -869,9 +931,36 @@ pub enum AttributeKind { /// Represents `#[rustc_layout_scalar_valid_range_start]`. RustcLayoutScalarValidRangeStart(Box, Span), + /// Represents `#[rustc_legacy_const_generics]` + RustcLegacyConstGenerics { fn_indexes: ThinVec<(usize, Span)>, attr_span: Span }, + + /// Represents `#[rustc_lint_diagnostics]` + RustcLintDiagnostics, + + /// Represents `#[rustc_lint_opt_deny_field_access]` + RustcLintOptDenyFieldAccess { lint_message: Symbol }, + + /// Represents `#[rustc_lint_opt_ty]` + RustcLintOptTy, + + /// Represents `#[rustc_lint_query_instability]` + RustcLintQueryInstability, + + /// Represents `#[rustc_lint_untracked_query_information]` + RustcLintUntrackedQueryInformation, + /// Represents `#[rustc_main]`. RustcMain, + /// Represents `#[rustc_must_implement_one_of]` + RustcMustImplementOneOf { attr_span: Span, fn_names: ThinVec }, + + /// Represents `#[rustc_never_returns_null_ptr]` + RustcNeverReturnsNullPointer, + + /// Represents `#[rustc_no_implicit_autorefs]` + RustcNoImplicitAutorefs, + /// Represents `#[rustc_object_lifetime_default]`. RustcObjectLifetimeDefault, @@ -927,6 +1016,9 @@ pub enum AttributeKind { /// `#[unsafe(force_target_feature(enable = "...")]`. TargetFeature { features: ThinVec<(Symbol, Span)>, attr_span: Span, was_forced: bool }, + /// Represents `#[thread_local]` + ThreadLocal, + /// Represents `#[track_caller]` TrackCaller(Span), diff --git a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs index 90fb0497a5b2..ce65350a2711 100644 --- a/compiler/rustc_hir/src/attrs/encode_cross_crate.rs +++ b/compiler/rustc_hir/src/attrs/encode_cross_crate.rs @@ -26,6 +26,7 @@ impl AttributeKind { AsPtr(..) => Yes, AutomaticallyDerived(..) => Yes, BodyStability { .. } => No, + CfiEncoding { .. } => Yes, Coinductive(..) => No, Cold(..) => No, Confusables { .. } => Yes, @@ -70,6 +71,7 @@ impl AttributeKind { Naked(..) => No, NoCore(..) => No, NoImplicitPrelude(..) => No, + NoLink => No, NoMangle(..) => Yes, // Needed for rustdoc NoStd(..) => No, NonExhaustive(..) => Yes, // Needed for rustdoc @@ -92,7 +94,16 @@ impl AttributeKind { RustcCoherenceIsCore(..) => No, RustcLayoutScalarValidRangeEnd(..) => Yes, RustcLayoutScalarValidRangeStart(..) => Yes, + RustcLegacyConstGenerics { .. } => Yes, + RustcLintDiagnostics => Yes, + RustcLintOptDenyFieldAccess { .. } => Yes, + RustcLintOptTy => Yes, + RustcLintQueryInstability => Yes, + RustcLintUntrackedQueryInformation => Yes, RustcMain => No, + RustcMustImplementOneOf { .. } => No, + RustcNeverReturnsNullPointer => Yes, + RustcNoImplicitAutorefs => Yes, RustcObjectLifetimeDefault => No, RustcPassIndirectlyInNonRusticAbis(..) => No, RustcScalableVector { .. } => Yes, @@ -105,6 +116,7 @@ impl AttributeKind { Stability { .. } => Yes, StdInternalSymbol(..) => No, TargetFeature { .. } => No, + ThreadLocal => No, TrackCaller(..) => Yes, TypeConst(..) => Yes, TypeLengthLimit { .. } => No, diff --git a/compiler/rustc_hir/src/attrs/pretty_printing.rs b/compiler/rustc_hir/src/attrs/pretty_printing.rs index 8bd267a1c256..f8ac2a547ca8 100644 --- a/compiler/rustc_hir/src/attrs/pretty_printing.rs +++ b/compiler/rustc_hir/src/attrs/pretty_printing.rs @@ -169,7 +169,7 @@ macro_rules! print_tup { print_tup!(A B C D E F G H); print_skip!(Span, (), ErrorGuaranteed); -print_disp!(u16, u128, bool, NonZero, Limit); +print_disp!(u16, u128, usize, bool, NonZero, Limit); print_debug!( Symbol, Ident, diff --git a/compiler/rustc_hir/src/hir.rs b/compiler/rustc_hir/src/hir.rs index c60471848c89..da3e79efd6df 100644 --- a/compiler/rustc_hir/src/hir.rs +++ b/compiler/rustc_hir/src/hir.rs @@ -150,10 +150,13 @@ impl From for LifetimeSyntax { /// `LifetimeSource::OutlivesBound` or `LifetimeSource::PreciseCapturing` /// — there's no way to "elide" these lifetimes. #[derive(Debug, Copy, Clone, HashStable_Generic)] -// Raise the aligement to at least 4 bytes - this is relied on in other parts of the compiler(for pointer tagging): -// https://github.com/rust-lang/rust/blob/ce5fdd7d42aba9a2925692e11af2bd39cf37798a/compiler/rustc_data_structures/src/tagged_ptr.rs#L163 -// Removing this `repr(4)` will cause the compiler to not build on platforms like `m68k` Linux, where the aligement of u32 and usize is only 2. -// Since `repr(align)` may only raise aligement, this has no effect on platforms where the aligement is already sufficient. +// Raise the alignment to at least 4 bytes. +// This is relied on in other parts of the compiler (for pointer tagging): +// +// Removing this `repr(4)` will cause the compiler to not build on platforms +// like `m68k` Linux, where the alignment of u32 and usize is only 2. +// Since `repr(align)` may only raise alignment, this has no effect on +// platforms where the alignment is already sufficient. #[repr(align(4))] pub struct Lifetime { #[stable_hasher(ignore)] @@ -494,6 +497,7 @@ impl<'hir, Unambig> ConstArg<'hir, Unambig> { pub fn span(&self) -> Span { match self.kind { + ConstArgKind::Struct(path, _) => path.span(), ConstArgKind::Path(path) => path.span(), ConstArgKind::Anon(anon) => anon.span, ConstArgKind::Error(span, _) => span, @@ -513,6 +517,8 @@ pub enum ConstArgKind<'hir, Unambig = ()> { /// However, in the future, we'll be using it for all of those. Path(QPath<'hir>), Anon(&'hir AnonConst), + /// Represents construction of struct/struct variants + Struct(QPath<'hir>, &'hir [&'hir ConstArgExprField<'hir>]), /// Error const Error(Span, ErrorGuaranteed), /// This variant is not always used to represent inference consts, sometimes @@ -520,6 +526,14 @@ pub enum ConstArgKind<'hir, Unambig = ()> { Infer(Span, Unambig), } +#[derive(Clone, Copy, Debug, HashStable_Generic)] +pub struct ConstArgExprField<'hir> { + pub hir_id: HirId, + pub span: Span, + pub field: Ident, + pub expr: &'hir ConstArg<'hir>, +} + #[derive(Clone, Copy, Debug, HashStable_Generic)] pub struct InferArg { #[stable_hasher(ignore)] @@ -1930,7 +1944,6 @@ pub enum PatExprKind<'hir> { // once instead of matching on unop neg expressions everywhere. negated: bool, }, - ConstBlock(ConstBlock), /// A path pattern for a unit struct/variant or a (maybe-associated) constant. Path(QPath<'hir>), } @@ -4714,6 +4727,7 @@ pub enum Node<'hir> { ConstArg(&'hir ConstArg<'hir>), Expr(&'hir Expr<'hir>), ExprField(&'hir ExprField<'hir>), + ConstArgExprField(&'hir ConstArgExprField<'hir>), Stmt(&'hir Stmt<'hir>), PathSegment(&'hir PathSegment<'hir>), Ty(&'hir Ty<'hir>), @@ -4773,6 +4787,7 @@ impl<'hir> Node<'hir> { Node::AssocItemConstraint(c) => Some(c.ident), Node::PatField(f) => Some(f.ident), Node::ExprField(f) => Some(f.ident), + Node::ConstArgExprField(f) => Some(f.field), Node::PreciseCapturingNonLifetimeArg(a) => Some(a.ident), Node::Param(..) | Node::AnonConst(..) diff --git a/compiler/rustc_hir/src/intravisit.rs b/compiler/rustc_hir/src/intravisit.rs index be3cab6461ef..e636b3fff654 100644 --- a/compiler/rustc_hir/src/intravisit.rs +++ b/compiler/rustc_hir/src/intravisit.rs @@ -396,6 +396,9 @@ pub trait Visitor<'v>: Sized { fn visit_expr_field(&mut self, field: &'v ExprField<'v>) -> Self::Result { walk_expr_field(self, field) } + fn visit_const_arg_expr_field(&mut self, field: &'v ConstArgExprField<'v>) -> Self::Result { + walk_const_arg_expr_field(self, field) + } fn visit_pattern_type_pattern(&mut self, p: &'v TyPat<'v>) -> Self::Result { walk_ty_pat(self, p) } @@ -789,7 +792,6 @@ pub fn walk_pat_expr<'v, V: Visitor<'v>>(visitor: &mut V, expr: &'v PatExpr<'v>) try_visit!(visitor.visit_id(*hir_id)); match kind { PatExprKind::Lit { lit, negated } => visitor.visit_lit(*hir_id, *lit, *negated), - PatExprKind::ConstBlock(c) => visitor.visit_inline_const(c), PatExprKind::Path(qpath) => visitor.visit_qpath(qpath, *hir_id, *span), } } @@ -954,6 +956,17 @@ pub fn walk_expr_field<'v, V: Visitor<'v>>(visitor: &mut V, field: &'v ExprField try_visit!(visitor.visit_ident(*ident)); visitor.visit_expr(*expr) } + +pub fn walk_const_arg_expr_field<'v, V: Visitor<'v>>( + visitor: &mut V, + field: &'v ConstArgExprField<'v>, +) -> V::Result { + let ConstArgExprField { hir_id, field, expr, span: _ } = field; + try_visit!(visitor.visit_id(*hir_id)); + try_visit!(visitor.visit_ident(*field)); + visitor.visit_const_arg_unambig(*expr) +} + /// We track whether an infer var is from a [`Ty`], [`ConstArg`], or [`GenericArg`] so that /// HIR visitors overriding [`Visitor::visit_infer`] can determine what kind of infer is being visited pub enum InferKind<'hir> { @@ -1068,6 +1081,15 @@ pub fn walk_const_arg<'v, V: Visitor<'v>>( let ConstArg { hir_id, kind } = const_arg; try_visit!(visitor.visit_id(*hir_id)); match kind { + ConstArgKind::Struct(qpath, field_exprs) => { + try_visit!(visitor.visit_qpath(qpath, *hir_id, qpath.span())); + + for field_expr in *field_exprs { + try_visit!(visitor.visit_const_arg_expr_field(field_expr)); + } + + V::Result::output() + } ConstArgKind::Path(qpath) => visitor.visit_qpath(qpath, *hir_id, qpath.span()), ConstArgKind::Anon(anon) => visitor.visit_anon_const(*anon), ConstArgKind::Error(_, _) => V::Result::output(), // errors and spans are not important diff --git a/compiler/rustc_hir/src/lib.rs b/compiler/rustc_hir/src/lib.rs index 7a5776f0d5a9..7c9c15c16df4 100644 --- a/compiler/rustc_hir/src/lib.rs +++ b/compiler/rustc_hir/src/lib.rs @@ -3,7 +3,6 @@ //! [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/hir.html // tidy-alphabetical-start -#![cfg_attr(bootstrap, feature(debug_closure_helpers))] #![feature(associated_type_defaults)] #![feature(closure_track_caller)] #![feature(const_default)] diff --git a/compiler/rustc_hir_analysis/README.md b/compiler/rustc_hir_analysis/README.md index b61dbd8c9648..c47271dbdf2b 100644 --- a/compiler/rustc_hir_analysis/README.md +++ b/compiler/rustc_hir_analysis/README.md @@ -1,5 +1,5 @@ For high-level intro to how type checking works in rustc, see the -[type checking] chapter of the [rustc dev guide]. +[hir typeck] chapter of the [rustc dev guide]. -[type checking]: https://rustc-dev-guide.rust-lang.org/type-checking.html +[hir typeck]: https://rustc-dev-guide.rust-lang.org/hir-typeck/summary.html [rustc dev guide]: https://rustc-dev-guide.rust-lang.org/ diff --git a/compiler/rustc_hir_analysis/messages.ftl b/compiler/rustc_hir_analysis/messages.ftl index 8022a48ee137..416a6b19edfc 100644 --- a/compiler/rustc_hir_analysis/messages.ftl +++ b/compiler/rustc_hir_analysis/messages.ftl @@ -208,14 +208,6 @@ hir_analysis_field_already_declared_previous_nested = .previous_decl_label = `{$field_name}` first declared here in this unnamed field .previous_nested_field_decl_note = field `{$field_name}` first declared here -hir_analysis_function_not_found_in_trait = function not found in this trait - -hir_analysis_function_not_have_default_implementation = function doesn't have a default implementation - .note = required by this annotation - -hir_analysis_functions_names_duplicated = functions names are duplicated - .note = all `#[rustc_must_implement_one_of]` arguments must be unique - hir_analysis_generic_args_on_overridden_impl = could not resolve generic parameters on overridden impl hir_analysis_impl_not_marked_default = `{$ident}` specializes an item from a parent `impl`, but that item is not marked `default` @@ -381,16 +373,6 @@ hir_analysis_missing_type_params = *[other] parameters } must be specified on the object type -hir_analysis_must_be_name_of_associated_function = must be a name of an associated function - -hir_analysis_must_implement_not_function = not a function - -hir_analysis_must_implement_not_function_note = all `#[rustc_must_implement_one_of]` arguments must be associated function names - -hir_analysis_must_implement_not_function_span_note = required by this annotation - -hir_analysis_must_implement_one_of_attribute = the `#[rustc_must_implement_one_of]` attribute must be used with at least 2 args - hir_analysis_no_variant_named = no variant named `{$ident}` found for enum `{$ty}` hir_analysis_not_supported_delegation = {$descr} diff --git a/compiler/rustc_hir_analysis/src/check/check.rs b/compiler/rustc_hir_analysis/src/check/check.rs index a81df02f023f..ad66003a6bf9 100644 --- a/compiler/rustc_hir_analysis/src/check/check.rs +++ b/compiler/rustc_hir_analysis/src/check/check.rs @@ -1340,9 +1340,7 @@ fn check_impl_items_against_trait<'tcx>( } if let Some(missing_items) = must_implement_one_of { - let attr_span = tcx - .get_attr(trait_ref.def_id, sym::rustc_must_implement_one_of) - .map(|attr| attr.span()); + let attr_span = find_attr!(tcx.get_all_attrs(trait_ref.def_id), AttributeKind::RustcMustImplementOneOf {attr_span, ..} => *attr_span); missing_items_must_implement_one_of_err( tcx, diff --git a/compiler/rustc_hir_analysis/src/check/wfcheck.rs b/compiler/rustc_hir_analysis/src/check/wfcheck.rs index 16f5222f621c..725294dfd377 100644 --- a/compiler/rustc_hir_analysis/src/check/wfcheck.rs +++ b/compiler/rustc_hir_analysis/src/check/wfcheck.rs @@ -327,7 +327,7 @@ pub(crate) fn check_trait_item<'tcx>( let mut res = Ok(()); - if matches!(tcx.def_kind(def_id), DefKind::AssocFn) { + if tcx.def_kind(def_id) == DefKind::AssocFn { for &assoc_ty_def_id in tcx.associated_types_for_impl_traits_in_associated_fn(def_id.to_def_id()) { diff --git a/compiler/rustc_hir_analysis/src/collect.rs b/compiler/rustc_hir_analysis/src/collect.rs index 2e06684e0c7a..9343bcd27a33 100644 --- a/compiler/rustc_hir_analysis/src/collect.rs +++ b/compiler/rustc_hir_analysis/src/collect.rs @@ -22,7 +22,6 @@ use std::ops::Bound; use rustc_abi::{ExternAbi, Size}; use rustc_ast::Recovered; use rustc_data_structures::fx::{FxHashSet, FxIndexMap}; -use rustc_data_structures::unord::UnordMap; use rustc_errors::{ Applicability, Diag, DiagCtxtHandle, E0228, ErrorGuaranteed, StashKey, struct_span_code_err, }; @@ -916,84 +915,15 @@ fn trait_def(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::TraitDef { } else { ty::trait_def::TraitSpecializationKind::None }; - let must_implement_one_of = attrs - .iter() - .find(|attr| attr.has_name(sym::rustc_must_implement_one_of)) - // Check that there are at least 2 arguments of `#[rustc_must_implement_one_of]` - // and that they are all identifiers - .and_then(|attr| match attr.meta_item_list() { - Some(items) if items.len() < 2 => { - tcx.dcx().emit_err(errors::MustImplementOneOfAttribute { span: attr.span() }); - None - } - Some(items) => items - .into_iter() - .map(|item| item.ident().ok_or(item.span())) - .collect::, _>>() - .map_err(|span| { - tcx.dcx().emit_err(errors::MustBeNameOfAssociatedFunction { span }); - }) - .ok() - .zip(Some(attr.span())), - // Error is reported by `rustc_attr!` - None => None, - }) - // Check that all arguments of `#[rustc_must_implement_one_of]` reference - // functions in the trait with default implementations - .and_then(|(list, attr_span)| { - let errors = list.iter().filter_map(|ident| { - let item = tcx - .associated_items(def_id) - .filter_by_name_unhygienic(ident.name) - .find(|item| item.ident(tcx) == *ident); - - match item { - Some(item) if matches!(item.kind, ty::AssocKind::Fn { .. }) => { - if !item.defaultness(tcx).has_value() { - tcx.dcx().emit_err(errors::FunctionNotHaveDefaultImplementation { - span: tcx.def_span(item.def_id), - note_span: attr_span, - }); - - return Some(()); - } - - return None; - } - Some(item) => { - tcx.dcx().emit_err(errors::MustImplementNotFunction { - span: tcx.def_span(item.def_id), - span_note: errors::MustImplementNotFunctionSpanNote { span: attr_span }, - note: errors::MustImplementNotFunctionNote {}, - }); - } - None => { - tcx.dcx().emit_err(errors::FunctionNotFoundInTrait { span: ident.span }); - } - } - - Some(()) - }); - - (errors.count() == 0).then_some(list) - }) - // Check for duplicates - .and_then(|list| { - let mut set: UnordMap = Default::default(); - let mut no_dups = true; - - for ident in &*list { - if let Some(dup) = set.insert(ident.name, ident.span) { - tcx.dcx() - .emit_err(errors::FunctionNamesDuplicated { spans: vec![dup, ident.span] }); - - no_dups = false; - } - } - - no_dups.then_some(list) - }); + let must_implement_one_of = find_attr!( + attrs, + AttributeKind::RustcMustImplementOneOf { fn_names, .. } => + fn_names + .iter() + .cloned() + .collect::>() + ); let deny_explicit_impl = find_attr!(attrs, AttributeKind::DenyExplicitImpl(_)); let implement_via_object = !find_attr!(attrs, AttributeKind::DoNotImplementViaObject(_)); diff --git a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs index 2eefe1eb3e92..178c47b09c84 100644 --- a/compiler/rustc_hir_analysis/src/collect/predicates_of.rs +++ b/compiler/rustc_hir_analysis/src/collect/predicates_of.rs @@ -543,7 +543,7 @@ pub(super) fn explicit_predicates_of<'tcx>( } } } else { - if matches!(def_kind, DefKind::AnonConst) + if def_kind == DefKind::AnonConst && tcx.features().generic_const_exprs() && let Some(defaulted_param_def_id) = tcx.hir_opt_const_param_default_param_def_id(tcx.local_def_id_to_hir_id(def_id)) diff --git a/compiler/rustc_hir_analysis/src/collect/type_of.rs b/compiler/rustc_hir_analysis/src/collect/type_of.rs index 180589340b4c..aa0e5c7fd710 100644 --- a/compiler/rustc_hir_analysis/src/collect/type_of.rs +++ b/compiler/rustc_hir_analysis/src/collect/type_of.rs @@ -10,6 +10,7 @@ use rustc_middle::ty::util::IntTypeExt; use rustc_middle::ty::{self, DefiningScopeKind, IsSuggestable, Ty, TyCtxt, TypeVisitableExt}; use rustc_middle::{bug, span_bug}; use rustc_span::{DUMMY_SP, Ident, Span}; +use tracing::instrument; use super::{HirPlaceholderCollector, ItemCtxt, bad_placeholder}; use crate::check::wfcheck::check_static_item; @@ -17,85 +18,7 @@ use crate::hir_ty_lowering::HirTyLowerer; mod opaque; -fn anon_const_type_of<'tcx>(icx: &ItemCtxt<'tcx>, def_id: LocalDefId) -> Ty<'tcx> { - use hir::*; - use rustc_middle::ty::Ty; - let tcx = icx.tcx; - let hir_id = tcx.local_def_id_to_hir_id(def_id); - - let node = tcx.hir_node(hir_id); - let Node::AnonConst(&AnonConst { span, .. }) = node else { - span_bug!( - tcx.def_span(def_id), - "expected anon const in `anon_const_type_of`, got {node:?}" - ); - }; - - let parent_node_id = tcx.parent_hir_id(hir_id); - let parent_node = tcx.hir_node(parent_node_id); - - match parent_node { - // Anon consts "inside" the type system. - Node::ConstArg(&ConstArg { - hir_id: arg_hir_id, - kind: ConstArgKind::Anon(&AnonConst { hir_id: anon_hir_id, .. }), - .. - }) if anon_hir_id == hir_id => const_arg_anon_type_of(icx, arg_hir_id, span), - - Node::Variant(Variant { disr_expr: Some(e), .. }) if e.hir_id == hir_id => { - tcx.adt_def(tcx.hir_get_parent_item(hir_id)).repr().discr_type().to_ty(tcx) - } - - Node::Field(&hir::FieldDef { default: Some(c), def_id: field_def_id, .. }) - if c.hir_id == hir_id => - { - tcx.type_of(field_def_id).instantiate_identity() - } - - _ => Ty::new_error_with_message( - tcx, - span, - format!("unexpected anon const parent in type_of(): {parent_node:?}"), - ), - } -} - -fn const_arg_anon_type_of<'tcx>(icx: &ItemCtxt<'tcx>, arg_hir_id: HirId, span: Span) -> Ty<'tcx> { - use hir::*; - use rustc_middle::ty::Ty; - - let tcx = icx.tcx; - - match tcx.parent_hir_node(arg_hir_id) { - // Array length const arguments do not have `type_of` fed as there is never a corresponding - // generic parameter definition. - Node::Ty(&hir::Ty { kind: TyKind::Array(_, ref constant), .. }) - | Node::Expr(&Expr { kind: ExprKind::Repeat(_, ref constant), .. }) - if constant.hir_id == arg_hir_id => - { - tcx.types.usize - } - - Node::TyPat(pat) => { - let node = match tcx.parent_hir_node(pat.hir_id) { - // Or patterns can be nested one level deep - Node::TyPat(p) => tcx.parent_hir_node(p.hir_id), - other => other, - }; - let hir::TyKind::Pat(ty, _) = node.expect_ty().kind else { bug!() }; - icx.lower_ty(ty) - } - - // This is not a `bug!` as const arguments in path segments that did not resolve to anything - // will result in `type_of` never being fed. - _ => Ty::new_error_with_message( - tcx, - span, - "`type_of` called on const argument's anon const before the const argument was lowered", - ), - } -} - +#[instrument(level = "debug", skip(tcx), ret)] pub(super) fn type_of(tcx: TyCtxt<'_>, def_id: LocalDefId) -> ty::EarlyBinder<'_, Ty<'_>> { use rustc_hir::*; use rustc_middle::ty::Ty; @@ -408,6 +331,85 @@ pub(super) fn type_of_opaque_hir_typeck( } } +fn anon_const_type_of<'tcx>(icx: &ItemCtxt<'tcx>, def_id: LocalDefId) -> Ty<'tcx> { + use hir::*; + use rustc_middle::ty::Ty; + let tcx = icx.tcx; + let hir_id = tcx.local_def_id_to_hir_id(def_id); + + let node = tcx.hir_node(hir_id); + let Node::AnonConst(&AnonConst { span, .. }) = node else { + span_bug!( + tcx.def_span(def_id), + "expected anon const in `anon_const_type_of`, got {node:?}" + ); + }; + + let parent_node_id = tcx.parent_hir_id(hir_id); + let parent_node = tcx.hir_node(parent_node_id); + + match parent_node { + // Anon consts "inside" the type system. + Node::ConstArg(&ConstArg { + hir_id: arg_hir_id, + kind: ConstArgKind::Anon(&AnonConst { hir_id: anon_hir_id, .. }), + .. + }) if anon_hir_id == hir_id => const_arg_anon_type_of(icx, arg_hir_id, span), + + Node::Variant(Variant { disr_expr: Some(e), .. }) if e.hir_id == hir_id => { + tcx.adt_def(tcx.hir_get_parent_item(hir_id)).repr().discr_type().to_ty(tcx) + } + + Node::Field(&hir::FieldDef { default: Some(c), def_id: field_def_id, .. }) + if c.hir_id == hir_id => + { + tcx.type_of(field_def_id).instantiate_identity() + } + + _ => Ty::new_error_with_message( + tcx, + span, + format!("unexpected anon const parent in type_of(): {parent_node:?}"), + ), + } +} + +fn const_arg_anon_type_of<'tcx>(icx: &ItemCtxt<'tcx>, arg_hir_id: HirId, span: Span) -> Ty<'tcx> { + use hir::*; + use rustc_middle::ty::Ty; + + let tcx = icx.tcx; + + match tcx.parent_hir_node(arg_hir_id) { + // Array length const arguments do not have `type_of` fed as there is never a corresponding + // generic parameter definition. + Node::Ty(&hir::Ty { kind: TyKind::Array(_, ref constant), .. }) + | Node::Expr(&Expr { kind: ExprKind::Repeat(_, ref constant), .. }) + if constant.hir_id == arg_hir_id => + { + tcx.types.usize + } + + Node::TyPat(pat) => { + let node = match tcx.parent_hir_node(pat.hir_id) { + // Or patterns can be nested one level deep + Node::TyPat(p) => tcx.parent_hir_node(p.hir_id), + other => other, + }; + let hir::TyKind::Pat(ty, _) = node.expect_ty().kind else { bug!() }; + icx.lower_ty(ty) + } + + // This is not a `bug!` as const arguments in path segments that did not resolve to anything + // will result in `type_of` never being fed. + _ => Ty::new_error_with_message( + tcx, + span, + "`type_of` called on const argument's anon const before the const argument was lowered", + ), + } +} + fn infer_placeholder_type<'tcx>( cx: &dyn HirTyLowerer<'tcx>, def_id: LocalDefId, diff --git a/compiler/rustc_hir_analysis/src/delegation.rs b/compiler/rustc_hir_analysis/src/delegation.rs index 125fc21a5cc1..4ab13140bf9c 100644 --- a/compiler/rustc_hir_analysis/src/delegation.rs +++ b/compiler/rustc_hir_analysis/src/delegation.rs @@ -401,12 +401,6 @@ fn check_constraints<'tcx>( })); }; - if let Some(local_sig_id) = sig_id.as_local() - && tcx.hir_opt_delegation_sig_id(local_sig_id).is_some() - { - emit("recursive delegation is not supported yet"); - } - if tcx.fn_sig(sig_id).skip_binder().skip_binder().c_variadic { // See issue #127443 for explanation. emit("delegation to C-variadic functions is not allowed"); diff --git a/compiler/rustc_hir_analysis/src/errors.rs b/compiler/rustc_hir_analysis/src/errors.rs index 8e17a4962520..b388396ac4fc 100644 --- a/compiler/rustc_hir_analysis/src/errors.rs +++ b/compiler/rustc_hir_analysis/src/errors.rs @@ -741,66 +741,6 @@ pub(crate) struct ParenSugarAttribute { pub span: Span, } -#[derive(Diagnostic)] -#[diag(hir_analysis_must_implement_one_of_attribute)] -pub(crate) struct MustImplementOneOfAttribute { - #[primary_span] - pub span: Span, -} - -#[derive(Diagnostic)] -#[diag(hir_analysis_must_be_name_of_associated_function)] -pub(crate) struct MustBeNameOfAssociatedFunction { - #[primary_span] - pub span: Span, -} - -#[derive(Diagnostic)] -#[diag(hir_analysis_function_not_have_default_implementation)] -pub(crate) struct FunctionNotHaveDefaultImplementation { - #[primary_span] - pub span: Span, - #[note] - pub note_span: Span, -} - -#[derive(Diagnostic)] -#[diag(hir_analysis_must_implement_not_function)] -pub(crate) struct MustImplementNotFunction { - #[primary_span] - pub span: Span, - #[subdiagnostic] - pub span_note: MustImplementNotFunctionSpanNote, - #[subdiagnostic] - pub note: MustImplementNotFunctionNote, -} - -#[derive(Subdiagnostic)] -#[note(hir_analysis_must_implement_not_function_span_note)] -pub(crate) struct MustImplementNotFunctionSpanNote { - #[primary_span] - pub span: Span, -} - -#[derive(Subdiagnostic)] -#[note(hir_analysis_must_implement_not_function_note)] -pub(crate) struct MustImplementNotFunctionNote {} - -#[derive(Diagnostic)] -#[diag(hir_analysis_function_not_found_in_trait)] -pub(crate) struct FunctionNotFoundInTrait { - #[primary_span] - pub span: Span, -} - -#[derive(Diagnostic)] -#[diag(hir_analysis_functions_names_duplicated)] -#[note] -pub(crate) struct FunctionNamesDuplicated { - #[primary_span] - pub spans: Vec, -} - #[derive(Diagnostic)] #[diag(hir_analysis_simd_ffi_highly_experimental)] #[help] diff --git a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs index 4db9a4c2bcdd..23110d2c5c87 100644 --- a/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs +++ b/compiler/rustc_hir_analysis/src/hir_ty_lowering/mod.rs @@ -2263,12 +2263,120 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ { ) .unwrap_or_else(|guar| Const::new_error(tcx, guar)) } - hir::ConstArgKind::Anon(anon) => self.lower_anon_const(anon), + hir::ConstArgKind::Struct(qpath, inits) => { + self.lower_const_arg_struct(hir_id, qpath, inits, const_arg.span()) + } + hir::ConstArgKind::Anon(anon) => self.lower_const_arg_anon(anon), hir::ConstArgKind::Infer(span, ()) => self.ct_infer(None, span), hir::ConstArgKind::Error(_, e) => ty::Const::new_error(tcx, e), } } + fn lower_const_arg_struct( + &self, + hir_id: HirId, + qpath: hir::QPath<'tcx>, + inits: &'tcx [&'tcx hir::ConstArgExprField<'tcx>], + span: Span, + ) -> Const<'tcx> { + // FIXME(mgca): try to deduplicate this function with + // the equivalent HIR typeck logic. + let tcx = self.tcx(); + + let non_adt_or_variant_res = || { + let e = tcx.dcx().span_err(span, "struct expression with invalid base path"); + ty::Const::new_error(tcx, e) + }; + + let (ty, variant_did) = match qpath { + hir::QPath::Resolved(maybe_qself, path) => { + debug!(?maybe_qself, ?path); + let opt_self_ty = maybe_qself.as_ref().map(|qself| self.lower_ty(qself)); + let ty = + self.lower_resolved_ty_path(opt_self_ty, path, hir_id, PermitVariants::Yes); + let variant_did = match path.res { + Res::Def(DefKind::Variant | DefKind::Struct, did) => did, + _ => return non_adt_or_variant_res(), + }; + + (ty, variant_did) + } + hir::QPath::TypeRelative(hir_self_ty, segment) => { + debug!(?hir_self_ty, ?segment); + let self_ty = self.lower_ty(hir_self_ty); + let opt_res = self.lower_type_relative_ty_path( + self_ty, + hir_self_ty, + segment, + hir_id, + span, + PermitVariants::Yes, + ); + + let (ty, _, res_def_id) = match opt_res { + Ok(r @ (_, DefKind::Variant | DefKind::Struct, _)) => r, + Ok(_) => return non_adt_or_variant_res(), + Err(e) => return ty::Const::new_error(tcx, e), + }; + + (ty, res_def_id) + } + }; + + let ty::Adt(adt_def, adt_args) = ty.kind() else { unreachable!() }; + + let variant_def = adt_def.variant_with_id(variant_did); + let variant_idx = adt_def.variant_index_with_id(variant_did).as_u32(); + + let fields = variant_def + .fields + .iter() + .map(|field_def| { + // FIXME(mgca): we aren't really handling privacy, stability, + // or macro hygeniene but we should. + let mut init_expr = + inits.iter().filter(|init_expr| init_expr.field.name == field_def.name); + + match init_expr.next() { + Some(expr) => { + if let Some(expr) = init_expr.next() { + let e = tcx.dcx().span_err( + expr.span, + format!( + "struct expression with multiple initialisers for `{}`", + field_def.name, + ), + ); + return ty::Const::new_error(tcx, e); + } + + self.lower_const_arg(expr.expr, FeedConstTy::Param(field_def.did, adt_args)) + } + None => { + let e = tcx.dcx().span_err( + span, + format!( + "struct expression with missing field initialiser for `{}`", + field_def.name + ), + ); + ty::Const::new_error(tcx, e) + } + } + }) + .collect::>(); + + let opt_discr_const = if adt_def.is_enum() { + let valtree = ty::ValTree::from_scalar_int(tcx, variant_idx.into()); + Some(ty::Const::new_value(tcx, valtree, tcx.types.u32)) + } else { + None + }; + + let valtree = ty::ValTree::from_branches(tcx, opt_discr_const.into_iter().chain(fields)); + ty::Const::new_value(tcx, valtree, ty) + } + /// Lower a [resolved][hir::QPath::Resolved] path to a (type-level) constant. fn lower_resolved_const_path( &self, @@ -2372,7 +2480,7 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ { /// Literals are eagerly converted to a constant, everything else becomes `Unevaluated`. #[instrument(skip(self), level = "debug")] - fn lower_anon_const(&self, anon: &AnonConst) -> Const<'tcx> { + fn lower_const_arg_anon(&self, anon: &AnonConst) -> Const<'tcx> { let tcx = self.tcx(); let expr = &tcx.hir_body(anon.body).value; @@ -2403,8 +2511,8 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ { ) -> Option> { let tcx = self.tcx(); - // Unwrap a block, so that e.g. `{ P }` is recognised as a parameter. Const arguments - // currently have to be wrapped in curly brackets, so it's necessary to special-case. + // Unwrap a block, so that e.g. `{ 1 }` is recognised as a literal. This makes the + // performance optimisation of directly lowering anon consts occur more often. let expr = match &expr.kind { hir::ExprKind::Block(block, _) if block.stmts.is_empty() && block.expr.is_some() => { block.expr.as_ref().unwrap() @@ -2412,15 +2520,18 @@ impl<'tcx> dyn HirTyLowerer<'tcx> + '_ { _ => expr, }; + // FIXME(mgca): remove this delayed bug once we start checking this + // when lowering `Ty/ConstKind::Param`s more generally. if let hir::ExprKind::Path(hir::QPath::Resolved( _, &hir::Path { res: Res::Def(DefKind::ConstParam, _), .. }, )) = expr.kind { - span_bug!( + let e = tcx.dcx().span_delayed_bug( expr.span, - "try_lower_anon_const_lit: received const param which shouldn't be possible" + "try_lower_anon_const_lit: received const param which shouldn't be possible", ); + return Some(ty::Const::new_error(tcx, e)); }; let lit_input = match expr.kind { diff --git a/compiler/rustc_hir_analysis/src/lib.rs b/compiler/rustc_hir_analysis/src/lib.rs index 538fb8c7df1e..65f6a11e8e9b 100644 --- a/compiler/rustc_hir_analysis/src/lib.rs +++ b/compiler/rustc_hir_analysis/src/lib.rs @@ -58,7 +58,6 @@ This API is completely unstable and subject to change. // tidy-alphabetical-start #![allow(rustc::diagnostic_outside_of_impl)] #![allow(rustc::untranslatable_diagnostic)] -#![cfg_attr(bootstrap, feature(debug_closure_helpers))] #![feature(assert_matches)] #![feature(gen_blocks)] #![feature(if_let_guard)] diff --git a/compiler/rustc_hir_pretty/src/lib.rs b/compiler/rustc_hir_pretty/src/lib.rs index b3b416955230..5c03135ee1bf 100644 --- a/compiler/rustc_hir_pretty/src/lib.rs +++ b/compiler/rustc_hir_pretty/src/lib.rs @@ -180,6 +180,8 @@ impl<'a> State<'a> { Node::ConstArg(a) => self.print_const_arg(a), Node::Expr(a) => self.print_expr(a), Node::ExprField(a) => self.print_expr_field(a), + // FIXME(mgca): proper printing for struct exprs + Node::ConstArgExprField(_) => self.word("/* STRUCT EXPR */"), Node::Stmt(a) => self.print_stmt(a), Node::PathSegment(a) => self.print_path_segment(a), Node::Ty(a) => self.print_type(a), @@ -1135,6 +1137,8 @@ impl<'a> State<'a> { fn print_const_arg(&mut self, const_arg: &hir::ConstArg<'_>) { match &const_arg.kind { + // FIXME(mgca): proper printing for struct exprs + ConstArgKind::Struct(..) => self.word("/* STRUCT EXPR */"), ConstArgKind::Path(qpath) => self.print_qpath(qpath, true), ConstArgKind::Anon(anon) => self.print_anon_const(anon), ConstArgKind::Error(_, _) => self.word("/*ERROR*/"), @@ -1876,7 +1880,6 @@ impl<'a> State<'a> { } self.print_literal(lit); } - hir::PatExprKind::ConstBlock(c) => self.print_inline_const(c), hir::PatExprKind::Path(qpath) => self.print_qpath(qpath, true), } } diff --git a/compiler/rustc_hir_typeck/src/_match.rs b/compiler/rustc_hir_typeck/src/_match.rs index 6467adb54dab..ded03c88a16e 100644 --- a/compiler/rustc_hir_typeck/src/_match.rs +++ b/compiler/rustc_hir_typeck/src/_match.rs @@ -10,7 +10,7 @@ use rustc_trait_selection::traits::{ }; use tracing::{debug, instrument}; -use crate::coercion::{AsCoercionSite, CoerceMany}; +use crate::coercion::CoerceMany; use crate::{Diverges, Expectation, FnCtxt, GatherLocalsVisitor, Needs}; impl<'a, 'tcx> FnCtxt<'a, 'tcx> { @@ -73,7 +73,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { Expectation::ExpectHasType(ety) if ety != tcx.types.unit => ety, _ => self.next_ty_var(expr.span), }; - CoerceMany::with_coercion_sites(coerce_first, arms) + CoerceMany::with_capacity(coerce_first, arms.len()) }; let mut prior_non_diverging_arms = vec![]; // Used only for diagnostics. @@ -269,16 +269,13 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { /// Handle the fallback arm of a desugared if(-let) like a missing else. /// /// Returns `true` if there was an error forcing the coercion to the `()` type. - pub(super) fn if_fallback_coercion( + pub(super) fn if_fallback_coercion( &self, if_span: Span, cond_expr: &'tcx hir::Expr<'tcx>, then_expr: &'tcx hir::Expr<'tcx>, - coercion: &mut CoerceMany<'tcx, '_, T>, - ) -> bool - where - T: AsCoercionSite, - { + coercion: &mut CoerceMany<'tcx>, + ) -> bool { // If this `if` expr is the parent's function return expr, // the cause of the type coercion is the return type, point at it. (#25228) let hir_id = self.tcx.parent_hir_id(self.tcx.parent_hir_id(then_expr.hir_id)); diff --git a/compiler/rustc_hir_typeck/src/coercion.rs b/compiler/rustc_hir_typeck/src/coercion.rs index 127965cb4b30..5e1e567d103e 100644 --- a/compiler/rustc_hir_typeck/src/coercion.rs +++ b/compiler/rustc_hir_typeck/src/coercion.rs @@ -1165,17 +1165,14 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { /// /// This is really an internal helper. From outside the coercion /// module, you should instantiate a `CoerceMany` instance. - fn try_find_coercion_lub( + fn try_find_coercion_lub( &self, cause: &ObligationCause<'tcx>, - exprs: &[E], + exprs: &[&'tcx hir::Expr<'tcx>], prev_ty: Ty<'tcx>, new: &hir::Expr<'_>, new_ty: Ty<'tcx>, - ) -> RelateResult<'tcx, Ty<'tcx>> - where - E: AsCoercionSite, - { + ) -> RelateResult<'tcx, Ty<'tcx>> { let prev_ty = self.try_structurally_resolve_type(cause.span, prev_ty); let new_ty = self.try_structurally_resolve_type(new.span, new_ty); debug!( @@ -1269,7 +1266,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { ty::FnDef(..) => Adjust::Pointer(PointerCoercion::ReifyFnPointer(sig.safety())), _ => span_bug!(new.span, "should not try to coerce a {new_ty} to a fn pointer"), }; - for expr in exprs.iter().map(|e| e.as_coercion_site()) { + for expr in exprs.iter() { self.apply_adjustments( expr, vec![Adjustment { kind: prev_adjustment.clone(), target: fn_ptr }], @@ -1316,7 +1313,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { let (adjustments, target) = self.register_infer_ok_obligations(ok); for expr in exprs { - let expr = expr.as_coercion_site(); self.apply_adjustments(expr, adjustments.clone()); } debug!( @@ -1382,41 +1378,23 @@ pub fn can_coerce<'tcx>( /// } /// let final_ty = coerce.complete(fcx); /// ``` -pub(crate) struct CoerceMany<'tcx, 'exprs, E: AsCoercionSite> { +pub(crate) struct CoerceMany<'tcx> { expected_ty: Ty<'tcx>, final_ty: Option>, - expressions: Expressions<'tcx, 'exprs, E>, - pushed: usize, + expressions: Vec<&'tcx hir::Expr<'tcx>>, } -/// The type of a `CoerceMany` that is storing up the expressions into -/// a buffer. We use this in `check/mod.rs` for things like `break`. -pub(crate) type DynamicCoerceMany<'tcx> = CoerceMany<'tcx, 'tcx, &'tcx hir::Expr<'tcx>>; - -enum Expressions<'tcx, 'exprs, E: AsCoercionSite> { - Dynamic(Vec<&'tcx hir::Expr<'tcx>>), - UpFront(&'exprs [E]), -} - -impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> { - /// The usual case; collect the set of expressions dynamically. - /// If the full set of coercion sites is known before hand, - /// consider `with_coercion_sites()` instead to avoid allocation. +impl<'tcx> CoerceMany<'tcx> { + /// Creates a `CoerceMany` with a default capacity of 1. If the full set of + /// coercion sites is known before hand, consider `with_capacity()` instead + /// to avoid allocation. pub(crate) fn new(expected_ty: Ty<'tcx>) -> Self { - Self::make(expected_ty, Expressions::Dynamic(vec![])) + Self::with_capacity(expected_ty, 1) } - /// As an optimization, you can create a `CoerceMany` with a - /// preexisting slice of expressions. In this case, you are - /// expected to pass each element in the slice to `coerce(...)` in - /// order. This is used with arrays in particular to avoid - /// needlessly cloning the slice. - pub(crate) fn with_coercion_sites(expected_ty: Ty<'tcx>, coercion_sites: &'exprs [E]) -> Self { - Self::make(expected_ty, Expressions::UpFront(coercion_sites)) - } - - fn make(expected_ty: Ty<'tcx>, expressions: Expressions<'tcx, 'exprs, E>) -> Self { - CoerceMany { expected_ty, final_ty: None, expressions, pushed: 0 } + /// Creates a `CoerceMany` with a given capacity. + pub(crate) fn with_capacity(expected_ty: Ty<'tcx>, capacity: usize) -> Self { + CoerceMany { expected_ty, final_ty: None, expressions: Vec::with_capacity(capacity) } } /// Returns the "expected type" with which this coercion was @@ -1529,7 +1507,7 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> { // Handle the actual type unification etc. let result = if let Some(expression) = expression { - if self.pushed == 0 { + if self.expressions.is_empty() { // Special-case the first expression we are coercing. // To be honest, I'm not entirely sure why we do this. // We don't allow two-phase borrows, see comment in try_find_coercion_lub for why @@ -1541,22 +1519,13 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> { Some(cause.clone()), ) } else { - match self.expressions { - Expressions::Dynamic(ref exprs) => fcx.try_find_coercion_lub( - cause, - exprs, - self.merged_ty(), - expression, - expression_ty, - ), - Expressions::UpFront(coercion_sites) => fcx.try_find_coercion_lub( - cause, - &coercion_sites[0..self.pushed], - self.merged_ty(), - expression, - expression_ty, - ), - } + fcx.try_find_coercion_lub( + cause, + &self.expressions, + self.merged_ty(), + expression, + expression_ty, + ) } } else { // this is a hack for cases where we default to `()` because @@ -1591,18 +1560,7 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> { Ok(v) => { self.final_ty = Some(v); if let Some(e) = expression { - match self.expressions { - Expressions::Dynamic(ref mut buffer) => buffer.push(e), - Expressions::UpFront(coercion_sites) => { - // if the user gave us an array to validate, check that we got - // the next expression in the list, as expected - assert_eq!( - coercion_sites[self.pushed].as_coercion_site().hir_id, - e.hir_id - ); - } - } - self.pushed += 1; + self.expressions.push(e); } } Err(coercion_error) => { @@ -1880,7 +1838,20 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> { hir::ExprKind::Match(.., hir::MatchSource::TryDesugar(_)) ) { - err.span_label(cond_expr.span, "expected this to be `()`"); + if let ObligationCauseCode::BlockTailExpression(hir_id, hir::MatchSource::Normal) = + cause.code() + && let hir::Node::Block(block) = fcx.tcx.hir_node(*hir_id) + && let hir::Node::Expr(expr) = fcx.tcx.parent_hir_node(block.hir_id) + && let hir::Node::Expr(if_expr) = fcx.tcx.parent_hir_node(expr.hir_id) + && let hir::ExprKind::If(_cond, _then, None) = if_expr.kind + { + err.span_label( + cond_expr.span, + "`if` expressions without `else` arms expect their inner expression to be `()`", + ); + } else { + err.span_label(cond_expr.span, "expected this to be `()`"); + } if expr.can_have_side_effects() { fcx.suggest_semicolon_at_end(cond_expr.span, &mut err); } @@ -1955,45 +1926,12 @@ impl<'tcx, 'exprs, E: AsCoercionSite> CoerceMany<'tcx, 'exprs, E> { } else { // If we only had inputs that were of type `!` (or no // inputs at all), then the final type is `!`. - assert_eq!(self.pushed, 0); + assert!(self.expressions.is_empty()); fcx.tcx.types.never } } } -/// Something that can be converted into an expression to which we can -/// apply a coercion. -pub(crate) trait AsCoercionSite { - fn as_coercion_site(&self) -> &hir::Expr<'_>; -} - -impl AsCoercionSite for hir::Expr<'_> { - fn as_coercion_site(&self) -> &hir::Expr<'_> { - self - } -} - -impl<'a, T> AsCoercionSite for &'a T -where - T: AsCoercionSite, -{ - fn as_coercion_site(&self) -> &hir::Expr<'_> { - (**self).as_coercion_site() - } -} - -impl AsCoercionSite for ! { - fn as_coercion_site(&self) -> &hir::Expr<'_> { - *self - } -} - -impl AsCoercionSite for hir::Arm<'_> { - fn as_coercion_site(&self) -> &hir::Expr<'_> { - self.body - } -} - /// Recursively visit goals to decide whether an unsizing is possible. /// `Break`s when it isn't, and an error should be raised. /// `Continue`s when an unsizing ok based on an implementation of the `Unsize` trait / lang item. diff --git a/compiler/rustc_hir_typeck/src/expr.rs b/compiler/rustc_hir_typeck/src/expr.rs index 48082560bb83..c4fa39c6c2c8 100644 --- a/compiler/rustc_hir_typeck/src/expr.rs +++ b/compiler/rustc_hir_typeck/src/expr.rs @@ -40,7 +40,7 @@ use tracing::{debug, instrument, trace}; use {rustc_ast as ast, rustc_hir as hir}; use crate::Expectation::{self, ExpectCastableToType, ExpectHasType, NoExpectation}; -use crate::coercion::{CoerceMany, DynamicCoerceMany}; +use crate::coercion::CoerceMany; use crate::errors::{ AddressOfTemporaryTaken, BaseExpressionDoubleDot, BaseExpressionDoubleDotAddExpr, BaseExpressionDoubleDotRemove, CantDereference, FieldMultiplySpecifiedInInitializer, @@ -1227,7 +1227,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { // (`only_has_type`); otherwise, we just go with a // fresh type variable. let coerce_to_ty = expected.coercion_target_type(self, sp); - let mut coerce: DynamicCoerceMany<'_> = CoerceMany::new(coerce_to_ty); + let mut coerce = CoerceMany::with_capacity(coerce_to_ty, 2); coerce.coerce(self, &self.misc(sp), then_expr, then_ty); @@ -1681,7 +1681,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { .to_option(self) .and_then(|uty| self.try_structurally_resolve_type(expr.span, uty).builtin_index()) .unwrap_or_else(|| self.next_ty_var(expr.span)); - let mut coerce = CoerceMany::with_coercion_sites(coerce_to, args); + let mut coerce = CoerceMany::with_capacity(coerce_to, args.len()); for e in args { let e_ty = self.check_expr_with_hint(e, coerce_to); diff --git a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs index 27b66d07f98e..ad34994526ed 100644 --- a/compiler/rustc_hir_typeck/src/expr_use_visitor.rs +++ b/compiler/rustc_hir_typeck/src/expr_use_visitor.rs @@ -7,9 +7,7 @@ use std::cell::{Ref, RefCell}; use std::ops::Deref; -use std::slice::from_ref; -use hir::Expr; use hir::def::DefKind; use hir::pat_util::EnumerateAndAdjustIterator as _; use rustc_abi::{FIRST_VARIANT, FieldIdx, VariantIdx}; @@ -23,6 +21,7 @@ use rustc_middle::hir::place::ProjectionKind; // Export these here so that Clippy can use them. pub use rustc_middle::hir::place::{Place, PlaceBase, PlaceWithHirId, Projection}; use rustc_middle::mir::FakeReadCause; +use rustc_middle::thir::DerefPatBorrowMode; use rustc_middle::ty::{ self, BorrowKind, Ty, TyCtxt, TypeFoldable, TypeVisitableExt as _, adjustment, }; @@ -313,7 +312,8 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx let param_place = self.cat_rvalue(param.hir_id, param_ty); - self.walk_irrefutable_pat(¶m_place, param.pat)?; + self.fake_read_scrutinee(¶m_place, false)?; + self.walk_pat(¶m_place, param.pat, false)?; } self.consume_expr(body.value)?; @@ -455,13 +455,9 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx hir::ExprKind::Match(discr, arms, _) => { let discr_place = self.cat_expr(discr)?; - self.maybe_read_scrutinee( - discr, - discr_place.clone(), - arms.iter().map(|arm| arm.pat), - )?; + self.fake_read_scrutinee(&discr_place, true)?; + self.walk_expr(discr)?; - // treatment of the discriminant is handled while walking the arms. for arm in arms { self.walk_arm(&discr_place, arm)?; } @@ -598,116 +594,25 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx Ok(()) } - fn maybe_read_scrutinee<'t>( + #[instrument(skip(self), level = "debug")] + fn fake_read_scrutinee( &self, - discr: &Expr<'_>, - discr_place: PlaceWithHirId<'tcx>, - pats: impl Iterator>, + discr_place: &PlaceWithHirId<'tcx>, + refutable: bool, ) -> Result<(), Cx::Error> { - // Matching should not always be considered a use of the place, hence - // discr does not necessarily need to be borrowed. - // We only want to borrow discr if the pattern contain something other - // than wildcards. - let mut needs_to_be_read = false; - for pat in pats { - self.cat_pattern(discr_place.clone(), pat, &mut |place, pat| { - match &pat.kind { - PatKind::Missing => unreachable!(), - PatKind::Binding(.., opt_sub_pat) => { - // If the opt_sub_pat is None, then the binding does not count as - // a wildcard for the purpose of borrowing discr. - if opt_sub_pat.is_none() { - needs_to_be_read = true; - } - } - PatKind::Never => { - // A never pattern reads the value. - // FIXME(never_patterns): does this do what I expect? - needs_to_be_read = true; - } - PatKind::Expr(PatExpr { kind: PatExprKind::Path(qpath), hir_id, span }) => { - // A `Path` pattern is just a name like `Foo`. This is either a - // named constant or else it refers to an ADT variant + let closure_def_id = match discr_place.place.base { + PlaceBase::Upvar(upvar_id) => Some(upvar_id.closure_expr_id), + _ => None, + }; - let res = self.cx.typeck_results().qpath_res(qpath, *hir_id); - match res { - Res::Def(DefKind::Const, _) | Res::Def(DefKind::AssocConst, _) => { - // Named constants have to be equated with the value - // being matched, so that's a read of the value being matched. - // - // FIXME: We don't actually reads for ZSTs. - needs_to_be_read = true; - } - _ => { - // Otherwise, this is a struct/enum variant, and so it's - // only a read if we need to read the discriminant. - needs_to_be_read |= - self.is_multivariant_adt(place.place.ty(), *span); - } - } - } - PatKind::TupleStruct(..) | PatKind::Struct(..) | PatKind::Tuple(..) => { - // For `Foo(..)`, `Foo { ... }` and `(...)` patterns, check if we are matching - // against a multivariant enum or struct. In that case, we have to read - // the discriminant. Otherwise this kind of pattern doesn't actually - // read anything (we'll get invoked for the `...`, which may indeed - // perform some reads). - - let place_ty = place.place.ty(); - needs_to_be_read |= self.is_multivariant_adt(place_ty, pat.span); - } - PatKind::Expr(_) | PatKind::Range(..) => { - // If the PatKind is a Lit or a Range then we want - // to borrow discr. - needs_to_be_read = true; - } - PatKind::Slice(lhs, wild, rhs) => { - // We don't need to test the length if the pattern is `[..]` - if matches!((lhs, wild, rhs), (&[], Some(_), &[])) - // Arrays have a statically known size, so - // there is no need to read their length - || place.place.ty().peel_refs().is_array() - { - } else { - needs_to_be_read = true; - } - } - PatKind::Or(_) - | PatKind::Box(_) - | PatKind::Deref(_) - | PatKind::Ref(..) - | PatKind::Guard(..) - | PatKind::Wild - | PatKind::Err(_) => { - // If the PatKind is Or, Box, or Ref, the decision is made later - // as these patterns contains subpatterns - // If the PatKind is Wild or Err, the decision is made based on the other patterns - // being examined - } - } - - Ok(()) - })? - } - - if needs_to_be_read { - self.borrow_expr(discr, BorrowKind::Immutable)?; + let cause = if refutable { + FakeReadCause::ForMatchedPlace(closure_def_id) } else { - let closure_def_id = match discr_place.place.base { - PlaceBase::Upvar(upvar_id) => Some(upvar_id.closure_expr_id), - _ => None, - }; + FakeReadCause::ForLet(closure_def_id) + }; - self.delegate.borrow_mut().fake_read( - &discr_place, - FakeReadCause::ForMatchedPlace(closure_def_id), - discr_place.hir_id, - ); + self.delegate.borrow_mut().fake_read(discr_place, cause, discr_place.hir_id); - // We always want to walk the discriminant. We want to make sure, for instance, - // that the discriminant has been initialized. - self.walk_expr(discr)?; - } Ok(()) } @@ -724,12 +629,11 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx self.walk_expr(expr)?; let expr_place = self.cat_expr(expr)?; f()?; + self.fake_read_scrutinee(&expr_place, els.is_some())?; + self.walk_pat(&expr_place, pat, false)?; if let Some(els) = els { - // borrowing because we need to test the discriminant - self.maybe_read_scrutinee(expr, expr_place.clone(), from_ref(pat).iter())?; self.walk_block(els)?; } - self.walk_irrefutable_pat(&expr_place, pat)?; Ok(()) } @@ -901,16 +805,6 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx discr_place: &PlaceWithHirId<'tcx>, arm: &hir::Arm<'_>, ) -> Result<(), Cx::Error> { - let closure_def_id = match discr_place.place.base { - PlaceBase::Upvar(upvar_id) => Some(upvar_id.closure_expr_id), - _ => None, - }; - - self.delegate.borrow_mut().fake_read( - discr_place, - FakeReadCause::ForMatchedPlace(closure_def_id), - discr_place.hir_id, - ); self.walk_pat(discr_place, arm.pat, arm.guard.is_some())?; if let Some(ref e) = arm.guard { @@ -921,28 +815,20 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx Ok(()) } - /// Walks a pat that occurs in isolation (i.e., top-level of fn argument or - /// let binding, and *not* a match arm or nested pat.) - fn walk_irrefutable_pat( - &self, - discr_place: &PlaceWithHirId<'tcx>, - pat: &hir::Pat<'_>, - ) -> Result<(), Cx::Error> { - let closure_def_id = match discr_place.place.base { - PlaceBase::Upvar(upvar_id) => Some(upvar_id.closure_expr_id), - _ => None, - }; - - self.delegate.borrow_mut().fake_read( - discr_place, - FakeReadCause::ForLet(closure_def_id), - discr_place.hir_id, - ); - self.walk_pat(discr_place, pat, false)?; - Ok(()) - } - /// The core driver for walking a pattern + /// + /// This should mirror how pattern-matching gets lowered to MIR, as + /// otherwise lowering will ICE when trying to resolve the upvars. + /// + /// However, it is okay to approximate it here by doing *more* accesses than + /// the actual MIR builder will, which is useful when some checks are too + /// cumbersome to perform here. For example, if after typeck it becomes + /// clear that only one variant of an enum is inhabited, and therefore a + /// read of the discriminant is not necessary, `walk_pat` will have + /// over-approximated the necessary upvar capture granularity. + /// + /// Do note that discrepancies like these do still create obscure corners + /// in the semantics of the language, and should be avoided if possible. #[instrument(skip(self), level = "debug")] fn walk_pat( &self, @@ -952,6 +838,11 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx ) -> Result<(), Cx::Error> { let tcx = self.cx.tcx(); self.cat_pattern(discr_place.clone(), pat, &mut |place, pat| { + debug!("walk_pat: pat.kind={:?}", pat.kind); + let read_discriminant = || { + self.delegate.borrow_mut().borrow(place, discr_place.hir_id, BorrowKind::Immutable); + }; + match pat.kind { PatKind::Binding(_, canonical_id, ..) => { debug!("walk_pat: binding place={:?} pat={:?}", place, pat); @@ -974,11 +865,7 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx // binding when lowering pattern guards to ensure that the guard does not // modify the scrutinee. if has_guard { - self.delegate.borrow_mut().borrow( - place, - discr_place.hir_id, - BorrowKind::Immutable, - ); + read_discriminant(); } // It is also a borrow or copy/move of the value being matched. @@ -1004,7 +891,7 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx // Deref patterns on boxes don't borrow, so we ignore them here. // HACK: this could be a fake pattern corresponding to a deref inserted by match // ergonomics, in which case `pat.hir_id` will be the id of the subpattern. - if let hir::ByRef::Yes(_, mutability) = + if let DerefPatBorrowMode::Borrow(mutability) = self.cx.typeck_results().deref_pat_borrow_mode(place.place.ty(), subpattern) { let bk = ty::BorrowKind::from_mutbl(mutability); @@ -1014,13 +901,73 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx PatKind::Never => { // A `!` pattern always counts as an immutable read of the discriminant, // even in an irrefutable pattern. - self.delegate.borrow_mut().borrow( - place, - discr_place.hir_id, - BorrowKind::Immutable, - ); + read_discriminant(); + } + PatKind::Expr(PatExpr { kind: PatExprKind::Path(qpath), hir_id, span }) => { + // A `Path` pattern is just a name like `Foo`. This is either a + // named constant or else it refers to an ADT variant + + let res = self.cx.typeck_results().qpath_res(qpath, *hir_id); + match res { + Res::Def(DefKind::Const, _) | Res::Def(DefKind::AssocConst, _) => { + // Named constants have to be equated with the value + // being matched, so that's a read of the value being matched. + // + // FIXME: Does the MIR code skip this read when matching on a ZST? + // If so, we can also skip it here. + read_discriminant(); + } + _ => { + // Otherwise, this is a struct/enum variant, and so it's + // only a read if we need to read the discriminant. + if self.is_multivariant_adt(place.place.ty(), *span) { + read_discriminant(); + } + } + } + } + PatKind::Expr(_) | PatKind::Range(..) => { + // When matching against a literal or range, we need to + // borrow the place to compare it against the pattern. + // + // Note that we do this read even if the range matches all + // possible values, such as 0..=u8::MAX. This is because + // we don't want to depend on consteval here. + // + // FIXME: What if the type being matched only has one + // possible value? + read_discriminant(); + } + PatKind::Struct(..) | PatKind::TupleStruct(..) => { + if self.is_multivariant_adt(place.place.ty(), pat.span) { + read_discriminant(); + } + } + PatKind::Slice(lhs, wild, rhs) => { + // We don't need to test the length if the pattern is `[..]` + if matches!((lhs, wild, rhs), (&[], Some(_), &[])) + // Arrays have a statically known size, so + // there is no need to read their length + || place.place.ty().peel_refs().is_array() + { + // No read necessary + } else { + read_discriminant(); + } + } + PatKind::Or(_) + | PatKind::Box(_) + | PatKind::Ref(..) + | PatKind::Guard(..) + | PatKind::Tuple(..) + | PatKind::Wild + | PatKind::Missing + | PatKind::Err(_) => { + // If the PatKind is Or, Box, Ref, Guard, or Tuple, the relevant accesses + // are made later as these patterns contains subpatterns. + // If the PatKind is Missing, Wild or Err, any relevant accesses are made when processing + // the other patterns that are part of the match } - _ => {} } Ok(()) @@ -1891,9 +1838,9 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx ) -> Result, Cx::Error> { match self.cx.typeck_results().deref_pat_borrow_mode(base_place.place.ty(), inner) { // Deref patterns on boxes are lowered using a built-in deref. - hir::ByRef::No => self.cat_deref(hir_id, base_place), + DerefPatBorrowMode::Box => self.cat_deref(hir_id, base_place), // For other types, we create a temporary to match on. - hir::ByRef::Yes(_, mutability) => { + DerefPatBorrowMode::Borrow(mutability) => { let re_erased = self.cx.tcx().lifetimes.re_erased; let ty = Ty::new_ref(self.cx.tcx(), re_erased, target_ty, mutability); // A deref pattern stores the result of `Deref::deref` or `DerefMut::deref_mut` ... @@ -1904,6 +1851,20 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx } } + /// Checks whether a type has multiple variants, and therefore, whether a + /// read of the discriminant might be necessary. Note that the actual MIR + /// builder code does a more specific check, filtering out variants that + /// happen to be uninhabited. + /// + /// Here, it is not practical to perform such a check, because inhabitedness + /// queries require typeck results, and typeck requires closure capture analysis. + /// + /// Moreover, the language is moving towards uninhabited variants still semantically + /// causing a discriminant read, so we *shouldn't* perform any such check. + /// + /// FIXME(never_patterns): update this comment once the aforementioned MIR builder + /// code is changed to be insensitive to inhhabitedness. + #[instrument(skip(self, span), level = "debug")] fn is_multivariant_adt(&self, ty: Ty<'tcx>, span: Span) -> bool { if let ty::Adt(def, _) = self.cx.structurally_resolve_type(span, ty).kind() { // Note that if a non-exhaustive SingleVariant is defined in another crate, we need diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs index da719e615fd7..ff85f53ddf30 100644 --- a/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs +++ b/compiler/rustc_hir_typeck/src/fn_ctxt/_impl.rs @@ -216,7 +216,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { // Don't write user type annotations for const param types, since we give them // identity args just so that we can trivially substitute their `EarlyBinder`. // We enforce that they match their type in MIR later on. - if matches!(self.tcx.def_kind(def_id), DefKind::ConstParam) { + if self.tcx.def_kind(def_id) == DefKind::ConstParam { return; } diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs index d04133ccee97..c07cbfae256d 100644 --- a/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs +++ b/compiler/rustc_hir_typeck/src/fn_ctxt/checks.rs @@ -1033,11 +1033,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { // break 'a 22; }` would not force the type of the block // to be `()`). let coerce_to_ty = expected.coercion_target_type(self, blk.span); - let coerce = if blk.targeted_by_break { - CoerceMany::new(coerce_to_ty) - } else { - CoerceMany::with_coercion_sites(coerce_to_ty, blk.expr.as_slice()) - }; + let coerce = CoerceMany::new(coerce_to_ty); let prev_diverges = self.diverges.get(); let ctxt = BreakableCtxt { coerce: Some(coerce), may_break: false }; diff --git a/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs b/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs index 998c5e6cd25a..c875e2e50d70 100644 --- a/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs +++ b/compiler/rustc_hir_typeck/src/fn_ctxt/mod.rs @@ -25,7 +25,7 @@ use rustc_trait_selection::traits::{ self, FulfillmentError, ObligationCause, ObligationCauseCode, ObligationCtxt, }; -use crate::coercion::DynamicCoerceMany; +use crate::coercion::CoerceMany; use crate::fallback::DivergingFallbackBehavior; use crate::fn_ctxt::checks::DivergingBlockBehavior; use crate::{CoroutineTypes, Diverges, EnclosingBreakables, TypeckRootCtxt}; @@ -56,13 +56,13 @@ pub(crate) struct FnCtxt<'a, 'tcx> { /// expressions. If `None`, this is in a context where return is /// inappropriate, such as a const expression. /// - /// This is a `RefCell`, which means that we + /// This is a `RefCell`, which means that we /// can track all the return expressions and then use them to /// compute a useful coercion from the set, similar to a match /// expression or other branching context. You can use methods /// like `expected_ty` to access the declared return type (if /// any). - pub(super) ret_coercion: Option>>, + pub(super) ret_coercion: Option>>, /// First span of a return site that we find. Used in error messages. pub(super) ret_coercion_span: Cell>, diff --git a/compiler/rustc_hir_typeck/src/lib.rs b/compiler/rustc_hir_typeck/src/lib.rs index d3ef1d63e8ba..b8a587016427 100644 --- a/compiler/rustc_hir_typeck/src/lib.rs +++ b/compiler/rustc_hir_typeck/src/lib.rs @@ -61,7 +61,7 @@ use tracing::{debug, instrument}; use typeck_root_ctxt::TypeckRootCtxt; use crate::check::check_fn; -use crate::coercion::DynamicCoerceMany; +use crate::coercion::CoerceMany; use crate::diverges::Diverges; use crate::expectation::Expectation; use crate::fn_ctxt::LoweredTy; @@ -115,6 +115,18 @@ fn typeck_with_inspect<'tcx>( return tcx.typeck(typeck_root_def_id); } + // We can't handle bodies containing generic parameters even though + // these generic parameters aren't part of its `generics_of` right now. + // + // See the FIXME on `check_anon_const_invalid_param_uses`. + if tcx.features().min_generic_const_args() + && let DefKind::AnonConst = tcx.def_kind(def_id) + && let ty::AnonConstKind::MCG = tcx.anon_const_kind(def_id) + && let Err(e) = tcx.check_anon_const_invalid_param_uses(def_id) + { + e.raise_fatal(); + } + let id = tcx.local_def_id_to_hir_id(def_id); let node = tcx.hir_node(id); let span = tcx.def_span(def_id); @@ -349,7 +361,7 @@ pub struct BreakableCtxt<'tcx> { // this is `null` for loops where break with a value is illegal, // such as `while`, `for`, and `while let` - coerce: Option>, + coerce: Option>, } pub struct EnclosingBreakables<'tcx> { diff --git a/compiler/rustc_hir_typeck/src/pat.rs b/compiler/rustc_hir_typeck/src/pat.rs index 06fd89837d51..1a579c4c6fa4 100644 --- a/compiler/rustc_hir_typeck/src/pat.rs +++ b/compiler/rustc_hir_typeck/src/pat.rs @@ -925,9 +925,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { } ty } - rustc_hir::PatExprKind::ConstBlock(c) => { - self.check_expr_const_block(c, Expectation::NoExpectation) - } rustc_hir::PatExprKind::Path(qpath) => { let (res, opt_ty, segments) = self.resolve_ty_and_res_fully_qualified_call(qpath, lt.hir_id, lt.span); diff --git a/compiler/rustc_hir_typeck/src/upvar.rs b/compiler/rustc_hir_typeck/src/upvar.rs index 445386412058..1a2b76485f35 100644 --- a/compiler/rustc_hir_typeck/src/upvar.rs +++ b/compiler/rustc_hir_typeck/src/upvar.rs @@ -761,6 +761,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> { /// ], /// } /// ``` + #[instrument(level = "debug", skip(self))] fn compute_min_captures( &self, closure_def_id: LocalDefId, @@ -2029,6 +2030,7 @@ struct InferBorrowKind<'tcx> { } impl<'tcx> euv::Delegate<'tcx> for InferBorrowKind<'tcx> { + #[instrument(skip(self), level = "debug")] fn fake_read( &mut self, place_with_id: &PlaceWithHirId<'tcx>, @@ -2119,6 +2121,7 @@ impl<'tcx> euv::Delegate<'tcx> for InferBorrowKind<'tcx> { } /// Rust doesn't permit moving fields out of a type that implements drop +#[instrument(skip(fcx), ret, level = "debug")] fn restrict_precision_for_drop_types<'a, 'tcx>( fcx: &'a FnCtxt<'a, 'tcx>, mut place: Place<'tcx>, @@ -2179,6 +2182,7 @@ fn restrict_precision_for_unsafe( /// - No unsafe block is required to capture `place`. /// /// Returns the truncated place and updated capture mode. +#[instrument(ret, level = "debug")] fn restrict_capture_precision( place: Place<'_>, curr_mode: ty::UpvarCapture, @@ -2208,6 +2212,7 @@ fn restrict_capture_precision( } /// Truncate deref of any reference. +#[instrument(ret, level = "debug")] fn adjust_for_move_closure( mut place: Place<'_>, mut kind: ty::UpvarCapture, @@ -2222,6 +2227,7 @@ fn adjust_for_move_closure( } /// Truncate deref of any reference. +#[instrument(ret, level = "debug")] fn adjust_for_use_closure( mut place: Place<'_>, mut kind: ty::UpvarCapture, @@ -2237,6 +2243,7 @@ fn adjust_for_use_closure( /// Adjust closure capture just that if taking ownership of data, only move data /// from enclosing stack frame. +#[instrument(ret, level = "debug")] fn adjust_for_non_move_closure( mut place: Place<'_>, mut kind: ty::UpvarCapture, @@ -2559,6 +2566,7 @@ fn determine_place_ancestry_relation<'tcx>( /// // it is constrained to `'a` /// } /// ``` +#[instrument(ret, level = "debug")] fn truncate_capture_for_optimization( mut place: Place<'_>, mut curr_mode: ty::UpvarCapture, diff --git a/compiler/rustc_index/src/slice.rs b/compiler/rustc_index/src/slice.rs index d2702bdb0571..415fe370b702 100644 --- a/compiler/rustc_index/src/slice.rs +++ b/compiler/rustc_index/src/slice.rs @@ -181,9 +181,6 @@ impl IndexSlice { /// Invert a bijective mapping, i.e. `invert(map)[y] = x` if `map[x] = y`, /// assuming the values in `self` are a permutation of `0..self.len()`. /// - /// This is used to go between `memory_index` (source field order to memory order) - /// and `inverse_memory_index` (memory order to source field order). - /// See also `FieldsShape::Arbitrary::memory_index` for more details. // FIXME(eddyb) build a better abstraction for permutations, if possible. pub fn invert_bijective_mapping(&self) -> IndexVec { debug_assert_eq!( diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 8dab3a7f37f5..d075f94ef850 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -837,7 +837,7 @@ fn test_unstable_options_tracking_hash() { tracked!(no_profiler_runtime, true); tracked!(no_trait_vptr, true); tracked!(no_unique_section_names, true); - tracked!(offload, vec![Offload::Enable]); + tracked!(offload, vec![Offload::Device]); tracked!(on_broken_pipe, OnBrokenPipe::Kill); tracked!(osx_rpath_install_name, true); tracked!(packed_bundled_libs, true); diff --git a/compiler/rustc_lexer/Cargo.toml b/compiler/rustc_lexer/Cargo.toml index 6d524b9d8873..057dce5b2f8d 100644 --- a/compiler/rustc_lexer/Cargo.toml +++ b/compiler/rustc_lexer/Cargo.toml @@ -15,8 +15,8 @@ Rust lexer used by rustc. No stability guarantees are provided. # Note that this crate purposefully does not depend on other rustc crates [dependencies] memchr = "2.7.6" -unicode-properties = { version = "0.1.0", default-features = false, features = ["emoji"] } -unicode-xid = "0.2.0" +unicode-properties = { version = "0.1.4", default-features = false, features = ["emoji"] } +unicode-ident = "1.0.22" [dev-dependencies] expect-test = "1.4.0" diff --git a/compiler/rustc_lexer/src/lib.rs b/compiler/rustc_lexer/src/lib.rs index f6790f7ed1e9..27ffcbc943bd 100644 --- a/compiler/rustc_lexer/src/lib.rs +++ b/compiler/rustc_lexer/src/lib.rs @@ -34,8 +34,25 @@ use LiteralKind::*; use TokenKind::*; use cursor::EOF_CHAR; pub use cursor::{Cursor, FrontmatterAllowed}; +pub use unicode_ident::UNICODE_VERSION; use unicode_properties::UnicodeEmoji; -pub use unicode_xid::UNICODE_VERSION as UNICODE_XID_VERSION; + +// Make sure that the Unicode version of the dependencies is the same. +const _: () = { + let properties = unicode_properties::UNICODE_VERSION; + let ident = unicode_ident::UNICODE_VERSION; + + if properties.0 != ident.0 as u64 + || properties.1 != ident.1 as u64 + || properties.2 != ident.2 as u64 + { + panic!( + "unicode-properties and unicode-ident must use the same Unicode version, \ + `unicode_properties::UNICODE_VERSION` and `unicode_ident::UNICODE_VERSION` are \ + different." + ); + } +}; /// Parsed token. /// It doesn't contain information about data that has been parsed, @@ -370,14 +387,14 @@ pub fn is_horizontal_whitespace(c: char) -> bool { /// a formal definition of valid identifier name. pub fn is_id_start(c: char) -> bool { // This is XID_Start OR '_' (which formally is not a XID_Start). - c == '_' || unicode_xid::UnicodeXID::is_xid_start(c) + c == '_' || unicode_ident::is_xid_start(c) } /// True if `c` is valid as a non-first character of an identifier. /// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for /// a formal definition of valid identifier name. pub fn is_id_continue(c: char) -> bool { - unicode_xid::UnicodeXID::is_xid_continue(c) + unicode_ident::is_xid_continue(c) } /// The passed string is lexically an identifier. diff --git a/compiler/rustc_lint/src/autorefs.rs b/compiler/rustc_lint/src/autorefs.rs index 5490a3aac9b7..ed7ac0e33244 100644 --- a/compiler/rustc_lint/src/autorefs.rs +++ b/compiler/rustc_lint/src/autorefs.rs @@ -1,8 +1,8 @@ use rustc_ast::{BorrowKind, UnOp}; -use rustc_hir::{Expr, ExprKind, Mutability}; +use rustc_hir::attrs::AttributeKind; +use rustc_hir::{Expr, ExprKind, Mutability, find_attr}; use rustc_middle::ty::adjustment::{Adjust, Adjustment, AutoBorrow, OverloadedDeref}; use rustc_session::{declare_lint, declare_lint_pass}; -use rustc_span::sym; use crate::lints::{ ImplicitUnsafeAutorefsDiag, ImplicitUnsafeAutorefsMethodNote, ImplicitUnsafeAutorefsOrigin, @@ -106,7 +106,7 @@ impl<'tcx> LateLintPass<'tcx> for ImplicitAutorefs { ExprKind::MethodCall(..) => cx.typeck_results().type_dependent_def_id(expr.hir_id), _ => None, } - && method_did.map(|did| cx.tcx.has_attr(did, sym::rustc_no_implicit_autorefs)).unwrap_or(true) + && method_did.map(|did| find_attr!(cx.tcx.get_all_attrs(did), AttributeKind::RustcNoImplicitAutorefs)).unwrap_or(true) { cx.emit_span_lint( DANGEROUS_IMPLICIT_AUTOREFS, diff --git a/compiler/rustc_lint/src/builtin.rs b/compiler/rustc_lint/src/builtin.rs index 810760e9f53e..e2a061cab680 100644 --- a/compiler/rustc_lint/src/builtin.rs +++ b/compiler/rustc_lint/src/builtin.rs @@ -2683,7 +2683,6 @@ declare_lint! { /// /// ```rust,compile_fail /// # #![allow(unused)] - /// # #![cfg_attr(bootstrap, deny(deref_nullptr))] /// use std::ptr; /// unsafe { /// let x = &*ptr::null::(); diff --git a/compiler/rustc_lint/src/interior_mutable_consts.rs b/compiler/rustc_lint/src/interior_mutable_consts.rs index 8576698dec33..4c7d2c6af93b 100644 --- a/compiler/rustc_lint/src/interior_mutable_consts.rs +++ b/compiler/rustc_lint/src/interior_mutable_consts.rs @@ -1,6 +1,7 @@ use rustc_hir::attrs::AttributeKind; use rustc_hir::def::{DefKind, Res}; use rustc_hir::{Expr, ExprKind, ItemKind, Node, find_attr}; +use rustc_middle::ty::adjustment::Adjust; use rustc_session::{declare_lint, declare_lint_pass}; use crate::lints::{ConstItemInteriorMutationsDiag, ConstItemInteriorMutationsSuggestionStatic}; @@ -77,6 +78,13 @@ impl<'tcx> LateLintPass<'tcx> for InteriorMutableConsts { if let ExprKind::Path(qpath) = &receiver.kind && let Res::Def(DefKind::Const | DefKind::AssocConst, const_did) = typeck.qpath_res(qpath, receiver.hir_id) + // Don't consider derefs as those can do arbitrary things + // like using thread local (see rust-lang/rust#150157) + && !cx + .typeck_results() + .expr_adjustments(receiver) + .into_iter() + .any(|adj| matches!(adj.kind, Adjust::Deref(_))) // Let's do the attribute check after the other checks for perf reasons && find_attr!( cx.tcx.get_all_attrs(method_did), diff --git a/compiler/rustc_lint/src/internal.rs b/compiler/rustc_lint/src/internal.rs index c00ba4959c94..68885e14f40d 100644 --- a/compiler/rustc_lint/src/internal.rs +++ b/compiler/rustc_lint/src/internal.rs @@ -1,9 +1,10 @@ //! Some lints that are only useful in the compiler or crates that use compiler internals, such as //! Clippy. +use rustc_hir::attrs::AttributeKind; use rustc_hir::def::Res; use rustc_hir::def_id::DefId; -use rustc_hir::{Expr, ExprKind, HirId}; +use rustc_hir::{Expr, ExprKind, HirId, find_attr}; use rustc_middle::ty::{self, GenericArgsRef, PredicatePolarity, Ty}; use rustc_session::{declare_lint_pass, declare_tool_lint}; use rustc_span::hygiene::{ExpnKind, MacroKind}; @@ -90,7 +91,7 @@ impl<'tcx> LateLintPass<'tcx> for QueryStability { ty::Instance::try_resolve(cx.tcx, cx.typing_env(), callee_def_id, generic_args) { let def_id = instance.def_id(); - if cx.tcx.has_attr(def_id, sym::rustc_lint_query_instability) { + if find_attr!(cx.tcx.get_all_attrs(def_id), AttributeKind::RustcLintQueryInstability) { cx.emit_span_lint( POTENTIAL_QUERY_INSTABILITY, span, @@ -105,7 +106,10 @@ impl<'tcx> LateLintPass<'tcx> for QueryStability { ); } - if cx.tcx.has_attr(def_id, sym::rustc_lint_untracked_query_information) { + if find_attr!( + cx.tcx.get_all_attrs(def_id), + AttributeKind::RustcLintUntrackedQueryInformation + ) { cx.emit_span_lint( UNTRACKED_QUERY_INFORMATION, span, @@ -150,7 +154,10 @@ fn has_unstable_into_iter_predicate<'tcx>( }; // Does the input type's `IntoIterator` implementation have the // `rustc_lint_query_instability` attribute on its `into_iter` method? - if cx.tcx.has_attr(instance.def_id(), sym::rustc_lint_query_instability) { + if find_attr!( + cx.tcx.get_all_attrs(instance.def_id()), + AttributeKind::RustcLintQueryInstability + ) { return true; } } @@ -602,14 +609,14 @@ impl Diagnostics { else { return; }; - let has_attr = cx.tcx.has_attr(inst.def_id(), sym::rustc_lint_diagnostics); - if !has_attr { + + if !find_attr!(cx.tcx.get_all_attrs(inst.def_id()), AttributeKind::RustcLintDiagnostics) { return; }; for (hir_id, _parent) in cx.tcx.hir_parent_iter(current_id) { if let Some(owner_did) = hir_id.as_owner() - && cx.tcx.has_attr(owner_did, sym::rustc_lint_diagnostics) + && find_attr!(cx.tcx.get_all_attrs(owner_did), AttributeKind::RustcLintDiagnostics) { // The parent method is marked with `#[rustc_lint_diagnostics]` return; @@ -658,23 +665,18 @@ impl LateLintPass<'_> for BadOptAccess { let Some(adt_def) = cx.typeck_results().expr_ty(base).ty_adt_def() else { return }; // Skip types without `#[rustc_lint_opt_ty]` - only so that the rest of the lint can be // avoided. - if !cx.tcx.has_attr(adt_def.did(), sym::rustc_lint_opt_ty) { + if !find_attr!(cx.tcx.get_all_attrs(adt_def.did()), AttributeKind::RustcLintOptTy) { return; } for field in adt_def.all_fields() { if field.name == target.name - && let Some(attr) = - cx.tcx.get_attr(field.did, sym::rustc_lint_opt_deny_field_access) - && let Some(items) = attr.meta_item_list() - && let Some(item) = items.first() - && let Some(lit) = item.lit() - && let ast::LitKind::Str(val, _) = lit.kind + && let Some(lint_message) = find_attr!(cx.tcx.get_all_attrs(field.did), AttributeKind::RustcLintOptDenyFieldAccess { lint_message, } => lint_message) { cx.emit_span_lint( BAD_OPT_ACCESS, expr.span, - BadOptAccessDiag { msg: val.as_str() }, + BadOptAccessDiag { msg: lint_message.as_str() }, ); } } diff --git a/compiler/rustc_lint/src/ptr_nulls.rs b/compiler/rustc_lint/src/ptr_nulls.rs index b2fa0fba76d9..b89e00dcbaae 100644 --- a/compiler/rustc_lint/src/ptr_nulls.rs +++ b/compiler/rustc_lint/src/ptr_nulls.rs @@ -1,5 +1,6 @@ use rustc_ast::LitKind; -use rustc_hir::{BinOpKind, Expr, ExprKind, TyKind}; +use rustc_hir::attrs::AttributeKind; +use rustc_hir::{BinOpKind, Expr, ExprKind, TyKind, find_attr}; use rustc_middle::ty::RawPtr; use rustc_session::{declare_lint, declare_lint_pass}; use rustc_span::{Span, sym}; @@ -72,14 +73,14 @@ fn useless_check<'a, 'tcx: 'a>( e = e.peel_blocks(); if let ExprKind::MethodCall(_, _expr, [], _) = e.kind && let Some(def_id) = cx.typeck_results().type_dependent_def_id(e.hir_id) - && cx.tcx.has_attr(def_id, sym::rustc_never_returns_null_ptr) + && find_attr!(cx.tcx.get_all_attrs(def_id), AttributeKind::RustcNeverReturnsNullPointer) && let Some(fn_name) = cx.tcx.opt_item_ident(def_id) { return Some(UselessPtrNullChecksDiag::FnRet { fn_name }); } else if let ExprKind::Call(path, _args) = e.kind && let ExprKind::Path(ref qpath) = path.kind && let Some(def_id) = cx.qpath_res(qpath, path.hir_id).opt_def_id() - && cx.tcx.has_attr(def_id, sym::rustc_never_returns_null_ptr) + && find_attr!(cx.tcx.get_all_attrs(def_id), AttributeKind::RustcNeverReturnsNullPointer) && let Some(fn_name) = cx.tcx.opt_item_ident(def_id) { return Some(UselessPtrNullChecksDiag::FnRet { fn_name }); diff --git a/compiler/rustc_lint_defs/src/builtin.rs b/compiler/rustc_lint_defs/src/builtin.rs index 99cce0c44b86..8c69cc089caf 100644 --- a/compiler/rustc_lint_defs/src/builtin.rs +++ b/compiler/rustc_lint_defs/src/builtin.rs @@ -2346,8 +2346,7 @@ declare_lint! { /// [sanitize]: https://doc.rust-lang.org/nightly/unstable-book/language-features/no-sanitize.html /// ### Example /// - #[cfg_attr(bootstrap, doc = "```ignore")] - #[cfg_attr(not(bootstrap), doc = "```rust,no_run")] + /// ```rust,no_run /// #![feature(sanitize)] /// /// #[sanitize(realtime = "nonblocking")] @@ -2356,8 +2355,7 @@ declare_lint! { /// fn main() { /// x(); /// } - #[cfg_attr(bootstrap, doc = "```")] - #[cfg_attr(not(bootstrap), doc = "```")] + /// ``` /// /// {{produces}} /// @@ -4907,8 +4905,7 @@ declare_lint! { /// /// ### Example /// - #[cfg_attr(bootstrap, doc = "```ignore")] - #[cfg_attr(not(bootstrap), doc = "```rust,compile_fail")] + /// ```rust,compile_fail /// #![feature(supertrait_item_shadowing)] /// #![deny(resolving_to_items_shadowing_supertrait_items)] /// @@ -4924,8 +4921,7 @@ declare_lint! { /// /// struct MyType; /// MyType.hello(); - #[cfg_attr(bootstrap, doc = "```")] - #[cfg_attr(not(bootstrap), doc = "```")] + /// ``` /// /// {{produces}} /// @@ -4951,8 +4947,7 @@ declare_lint! { /// /// ### Example /// - #[cfg_attr(bootstrap, doc = "```ignore")] - #[cfg_attr(not(bootstrap), doc = "```rust,compile_fail")] + /// ```rust,compile_fail /// #![feature(supertrait_item_shadowing)] /// #![deny(shadowing_supertrait_items)] /// @@ -4965,8 +4960,7 @@ declare_lint! { /// fn hello(&self) {} /// } /// impl Downstream for T {} - #[cfg_attr(bootstrap, doc = "```")] - #[cfg_attr(not(bootstrap), doc = "```")] + /// ``` /// /// {{produces}} /// diff --git a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp index 0720af0eb7e0..3cc2610c5077 100644 --- a/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp +++ b/compiler/rustc_llvm/llvm-wrapper/RustWrapper.cpp @@ -43,8 +43,10 @@ // available. As such, we only try to build it in the first place, if // llvm.offload is enabled. #ifdef OFFLOAD +#include "llvm/Bitcode/BitcodeReader.h" #include "llvm/Object/OffloadBinary.h" #include "llvm/Target/TargetMachine.h" +#include "llvm/Transforms/Utils/ModuleUtils.h" #endif // for raw `write` in the bad-alloc handler @@ -174,12 +176,13 @@ static Error writeFile(StringRef Filename, StringRef Data) { // --image=file=device.bc,triple=amdgcn-amd-amdhsa,arch=gfx90a,kind=openmp // The input module is the rust code compiled for a gpu target like amdgpu. // Based on clang/tools/clang-offload-packager/ClangOffloadPackager.cpp -extern "C" bool LLVMRustBundleImages(LLVMModuleRef M, TargetMachine &TM) { +extern "C" bool LLVMRustBundleImages(LLVMModuleRef M, TargetMachine &TM, + const char *HostOutPath) { std::string Storage; llvm::raw_string_ostream OS1(Storage); llvm::WriteBitcodeToFile(*unwrap(M), OS1); OS1.flush(); - auto MB = llvm::MemoryBuffer::getMemBufferCopy(Storage, "module.bc"); + auto MB = llvm::MemoryBuffer::getMemBufferCopy(Storage, "device.bc"); SmallVector BinaryData; raw_svector_ostream OS2(BinaryData); @@ -188,19 +191,38 @@ extern "C" bool LLVMRustBundleImages(LLVMModuleRef M, TargetMachine &TM) { ImageBinary.TheImageKind = object::IMG_Bitcode; ImageBinary.Image = std::move(MB); ImageBinary.TheOffloadKind = object::OFK_OpenMP; - ImageBinary.StringData["triple"] = TM.getTargetTriple().str(); - ImageBinary.StringData["arch"] = TM.getTargetCPU(); + + std::string TripleStr = TM.getTargetTriple().str(); + llvm::StringRef CPURef = TM.getTargetCPU(); + ImageBinary.StringData["triple"] = TripleStr; + ImageBinary.StringData["arch"] = CPURef; llvm::SmallString<0> Buffer = OffloadBinary::write(ImageBinary); if (Buffer.size() % OffloadBinary::getAlignment() != 0) // Offload binary has invalid size alignment return false; OS2 << Buffer; - if (Error E = writeFile("host.out", + if (Error E = writeFile(HostOutPath, StringRef(BinaryData.begin(), BinaryData.size()))) return false; return true; } +extern "C" bool LLVMRustOffloadEmbedBufferInModule(LLVMModuleRef HostM, + const char *HostOutPath) { + auto MBOrErr = MemoryBuffer::getFile(HostOutPath); + if (!MBOrErr) { + auto E = MBOrErr.getError(); + auto _B = errorCodeToError(E); + return false; + } + MemoryBufferRef Buf = (*MBOrErr)->getMemBufferRef(); + Module *M = unwrap(HostM); + StringRef SectionName = ".llvm.offloading"; + Align Alignment = Align(8); + llvm::embedBufferInModule(*M, Buf, SectionName, Alignment); + return true; +} + extern "C" void LLVMRustOffloadMapper(LLVMValueRef OldFn, LLVMValueRef NewFn) { llvm::Function *oldFn = llvm::unwrap(OldFn); llvm::Function *newFn = llvm::unwrap(NewFn); @@ -1550,16 +1572,8 @@ extern "C" uint64_t LLVMRustModuleCost(LLVMModuleRef M) { return std::distance(std::begin(f), std::end(f)); } -extern "C" void LLVMRustModuleInstructionStats(LLVMModuleRef M, - RustStringRef Str) { - auto OS = RawRustStringOstream(Str); - auto JOS = llvm::json::OStream(OS); - auto Module = unwrap(M); - - JOS.object([&] { - JOS.attribute("module", Module->getName()); - JOS.attribute("total", Module->getInstructionCount()); - }); +extern "C" uint64_t LLVMRustModuleInstructionStats(LLVMModuleRef M) { + return unwrap(M)->getInstructionCount(); } // Transfers ownership of DiagnosticHandler unique_ptr to the caller. diff --git a/compiler/rustc_metadata/src/errors.rs b/compiler/rustc_metadata/src/errors.rs index abfd078f7462..2702c4498df6 100644 --- a/compiler/rustc_metadata/src/errors.rs +++ b/compiler/rustc_metadata/src/errors.rs @@ -251,9 +251,7 @@ impl<'a> MissingNativeLibrary<'a> { // if it looks like the user has provided a complete filename rather just the bare lib name, // then provide a note that they might want to try trimming the name let suggested_name = if !verbatim { - if let Some(libname) = libname.strip_prefix("lib") - && let Some(libname) = libname.strip_suffix(".a") - { + if let Some(libname) = libname.strip_circumfix("lib", ".a") { // this is a unix style filename so trim prefix & suffix Some(libname) } else if let Some(libname) = libname.strip_suffix(".lib") { @@ -466,6 +464,7 @@ pub struct CannotFindCrate { pub profiler_runtime: Symbol, pub locator_triple: TargetTuple, pub is_ui_testing: bool, + pub is_tier_3: bool, } impl Diagnostic<'_, G> for CannotFindCrate { @@ -485,11 +484,13 @@ impl Diagnostic<'_, G> for CannotFindCrate { diag.note(fluent::metadata_target_no_std_support); } + let has_precompiled_std = !self.is_tier_3; + if self.missing_core { if env!("CFG_RELEASE_CHANNEL") == "dev" && !self.is_ui_testing { // Note: Emits the nicer suggestion only for the dev channel. diag.help(fluent::metadata_consider_adding_std); - } else { + } else if has_precompiled_std { // NOTE: this suggests using rustup, even though the user may not have it installed. // That's because they could choose to install it; or this may give them a hint which // target they need to install from their distro. @@ -504,7 +505,9 @@ impl Diagnostic<'_, G> for CannotFindCrate { if !self.missing_core && self.span.is_dummy() { diag.note(fluent::metadata_std_required); } - if self.is_nightly_build { + // Recommend -Zbuild-std even on stable builds for Tier 3 targets because + // it's the recommended way to use the target, the user should switch to nightly. + if self.is_nightly_build || !has_precompiled_std { diag.help(fluent::metadata_consider_building_std); } } else if self.crate_name == self.profiler_runtime { diff --git a/compiler/rustc_metadata/src/lib.rs b/compiler/rustc_metadata/src/lib.rs index f31c4938e118..f3b738f93d2d 100644 --- a/compiler/rustc_metadata/src/lib.rs +++ b/compiler/rustc_metadata/src/lib.rs @@ -10,6 +10,7 @@ #![feature(never_type)] #![feature(proc_macro_internals)] #![feature(result_option_map_or_default)] +#![feature(strip_circumfix)] #![feature(trusted_len)] // tidy-alphabetical-end diff --git a/compiler/rustc_metadata/src/locator.rs b/compiler/rustc_metadata/src/locator.rs index c4083a27e72b..6c30ce0ddb3d 100644 --- a/compiler/rustc_metadata/src/locator.rs +++ b/compiler/rustc_metadata/src/locator.rs @@ -1224,6 +1224,7 @@ impl CrateError { profiler_runtime: Symbol::intern(&sess.opts.unstable_opts.profiler_runtime), locator_triple: locator.triple, is_ui_testing: sess.opts.unstable_opts.ui_testing, + is_tier_3: sess.target.metadata.tier == Some(3), }; // The diagnostic for missing core is very good, but it is followed by a lot of // other diagnostics that do not add information. @@ -1249,6 +1250,7 @@ impl CrateError { profiler_runtime: Symbol::intern(&sess.opts.unstable_opts.profiler_runtime), locator_triple: sess.opts.target_triple.clone(), is_ui_testing: sess.opts.unstable_opts.ui_testing, + is_tier_3: sess.target.metadata.tier == Some(3), }; // The diagnostic for missing core is very good, but it is followed by a lot of // other diagnostics that do not add information. diff --git a/compiler/rustc_metadata/src/rmeta/decoder.rs b/compiler/rustc_metadata/src/rmeta/decoder.rs index e3c2113fe71c..f8610f0c5b03 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder.rs @@ -1675,15 +1675,14 @@ impl<'a> CrateMetadataRef<'a> { for virtual_dir in virtual_source_base_dir.iter().flatten() { if let Some(real_dir) = &real_source_base_dir && let rustc_span::FileName::Real(old_name) = name - && let (_working_dir, embeddable_name) = - old_name.embeddable_name(RemapPathScopeComponents::MACRO) - && let Ok(rest) = embeddable_name.strip_prefix(virtual_dir) + && let virtual_path = old_name.path(RemapPathScopeComponents::MACRO) + && let Ok(rest) = virtual_path.strip_prefix(virtual_dir) { let new_path = real_dir.join(rest); debug!( "try_to_translate_virtual_to_real: `{}` -> `{}`", - embeddable_name.display(), + virtual_path.display(), new_path.display(), ); diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 7da82835befb..920c896d5a47 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -879,13 +879,9 @@ fn analyze_attr(attr: &hir::Attribute, state: &mut AnalyzeAttrState<'_>) -> bool should_encode = true; } } else if let hir::Attribute::Parsed(AttributeKind::Doc(d)) = attr { - // If this is a `doc` attribute that doesn't have anything except maybe `inline` (as in - // `#[doc(inline)]`), then we can remove it. It won't be inlinable in downstream crates. - if d.inline.is_empty() { - should_encode = true; - if d.hidden.is_some() { - state.is_doc_hidden = true; - } + should_encode = true; + if d.hidden.is_some() { + state.is_doc_hidden = true; } } else if let &[sym::diagnostic, seg] = &*attr.path() { should_encode = rustc_feature::is_stable_diagnostic_attribute(seg, state.features); @@ -1444,6 +1440,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> { hir::Node::ConstArg(hir::ConstArg { kind, .. }) => match kind { // Skip encoding defs for these as they should not have had a `DefId` created hir::ConstArgKind::Error(..) + | hir::ConstArgKind::Struct(..) | hir::ConstArgKind::Path(..) | hir::ConstArgKind::Infer(..) => true, hir::ConstArgKind::Anon(..) => false, diff --git a/compiler/rustc_middle/src/arena.rs b/compiler/rustc_middle/src/arena.rs index d1d4c32184ee..4fa39eb83e9e 100644 --- a/compiler/rustc_middle/src/arena.rs +++ b/compiler/rustc_middle/src/arena.rs @@ -92,7 +92,7 @@ macro_rules! arena_types { [] name_set: rustc_data_structures::unord::UnordSet, [] autodiff_item: rustc_ast::expand::autodiff_attrs::AutoDiffItem, [] ordered_name_set: rustc_data_structures::fx::FxIndexSet, - [] valtree: rustc_middle::ty::ValTreeKind<'tcx>, + [] valtree: rustc_middle::ty::ValTreeKind>, [] stable_order_of_exportable_impls: rustc_data_structures::fx::FxIndexMap, diff --git a/compiler/rustc_middle/src/hir/map.rs b/compiler/rustc_middle/src/hir/map.rs index 5da762ef8565..d62ddc915d17 100644 --- a/compiler/rustc_middle/src/hir/map.rs +++ b/compiler/rustc_middle/src/hir/map.rs @@ -2,6 +2,8 @@ //! eliminated, and all its methods are now on `TyCtxt`. But the module name //! stays as `map` because there isn't an obviously better name for it. +use std::ops::ControlFlow; + use rustc_abi::ExternAbi; use rustc_ast::visit::{VisitorResult, walk_list}; use rustc_data_structures::fingerprint::Fingerprint; @@ -737,6 +739,7 @@ impl<'tcx> TyCtxt<'tcx> { Node::ConstArg(_) => node_str("const"), Node::Expr(_) => node_str("expr"), Node::ExprField(_) => node_str("expr field"), + Node::ConstArgExprField(_) => node_str("const arg expr field"), Node::Stmt(_) => node_str("stmt"), Node::PathSegment(_) => node_str("path segment"), Node::Ty(_) => node_str("type"), @@ -1005,6 +1008,7 @@ impl<'tcx> TyCtxt<'tcx> { Node::ConstArg(const_arg) => const_arg.span(), Node::Expr(expr) => expr.span, Node::ExprField(field) => field.span, + Node::ConstArgExprField(field) => field.span, Node::Stmt(stmt) => stmt.span, Node::PathSegment(seg) => { let ident_span = seg.ident.span; @@ -1086,6 +1090,52 @@ impl<'tcx> TyCtxt<'tcx> { None } + + // FIXME(mgca): this is pretty iffy. In the long term we should make + // HIR ty lowering able to return `Error` versions of types/consts when + // lowering them in contexts that aren't supposed to use generic parameters. + // + // This current impl strategy is incomplete and doesn't handle `Self` ty aliases. + pub fn check_anon_const_invalid_param_uses( + self, + anon: LocalDefId, + ) -> Result<(), ErrorGuaranteed> { + struct GenericParamVisitor<'tcx>(TyCtxt<'tcx>); + impl<'tcx> Visitor<'tcx> for GenericParamVisitor<'tcx> { + type NestedFilter = nested_filter::OnlyBodies; + type Result = ControlFlow; + + fn maybe_tcx(&mut self) -> TyCtxt<'tcx> { + self.0 + } + + fn visit_path( + &mut self, + path: &crate::hir::Path<'tcx>, + _id: HirId, + ) -> ControlFlow { + if let Res::Def( + DefKind::TyParam | DefKind::ConstParam | DefKind::LifetimeParam, + _, + ) = path.res + { + let e = self.0.dcx().struct_span_err( + path.span, + "generic parameters may not be used in const operations", + ); + return ControlFlow::Break(e.emit()); + } + + intravisit::walk_path(self, path) + } + } + + let body = self.hir_maybe_body_owned_by(anon).unwrap(); + match GenericParamVisitor(self).visit_expr(&body.value) { + ControlFlow::Break(e) => Err(e), + ControlFlow::Continue(()) => Ok(()), + } + } } impl<'tcx> intravisit::HirTyCtxt<'tcx> for TyCtxt<'tcx> { diff --git a/compiler/rustc_middle/src/hir/mod.rs b/compiler/rustc_middle/src/hir/mod.rs index 217ecbab059e..ba2d8febad7c 100644 --- a/compiler/rustc_middle/src/hir/mod.rs +++ b/compiler/rustc_middle/src/hir/mod.rs @@ -359,6 +359,7 @@ impl<'tcx> TyCtxt<'tcx> { | Node::Infer(_) | Node::WherePredicate(_) | Node::PreciseCapturingNonLifetimeArg(_) + | Node::ConstArgExprField(_) | Node::OpaqueTy(_) => { unreachable!("no sub-expr expected for {parent_node:?}") } diff --git a/compiler/rustc_middle/src/lint.rs b/compiler/rustc_middle/src/lint.rs index bed902e8334b..e4715f6e2c10 100644 --- a/compiler/rustc_middle/src/lint.rs +++ b/compiler/rustc_middle/src/lint.rs @@ -265,9 +265,16 @@ fn explain_lint_level_source( "`{flag} {hyphen_case_lint_name}` implied by `{flag} {hyphen_case_flag_val}`" )); if matches!(orig_level, Level::Warn | Level::Deny) { - err.help_once(format!( - "to override `{flag} {hyphen_case_flag_val}` add `#[allow({name})]`" - )); + let help = if name == "dead_code" { + format!( + "to override `{flag} {hyphen_case_flag_val}` add `#[expect({name})]` or `#[allow({name})]`" + ) + } else { + format!( + "to override `{flag} {hyphen_case_flag_val}` add `#[allow({name})]`" + ) + }; + err.help_once(help); } } } diff --git a/compiler/rustc_middle/src/mir/consts.rs b/compiler/rustc_middle/src/mir/consts.rs index fe352df3b9f0..afe39e4481ef 100644 --- a/compiler/rustc_middle/src/mir/consts.rs +++ b/compiler/rustc_middle/src/mir/consts.rs @@ -302,15 +302,7 @@ impl<'tcx> Const<'tcx> { #[inline] pub fn try_to_scalar(self) -> Option { match self { - Const::Ty(_, c) => match c.kind() { - ty::ConstKind::Value(cv) if cv.ty.is_primitive() => { - // A valtree of a type where leaves directly represent the scalar const value. - // Just checking whether it is a leaf is insufficient as e.g. references are leafs - // but the leaf value is the value they point to, not the reference itself! - Some(cv.valtree.unwrap_leaf().into()) - } - _ => None, - }, + Const::Ty(_, c) => c.try_to_scalar(), Const::Val(val, _) => val.try_to_scalar(), Const::Unevaluated(..) => None, } @@ -321,10 +313,7 @@ impl<'tcx> Const<'tcx> { // This is equivalent to `self.try_to_scalar()?.try_to_int().ok()`, but measurably faster. match self { Const::Val(ConstValue::Scalar(Scalar::Int(x)), _) => Some(x), - Const::Ty(_, c) => match c.kind() { - ty::ConstKind::Value(cv) if cv.ty.is_primitive() => Some(cv.valtree.unwrap_leaf()), - _ => None, - }, + Const::Ty(_, c) => c.try_to_leaf(), _ => None, } } @@ -377,14 +366,10 @@ impl<'tcx> Const<'tcx> { tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>, ) -> Option { - if let Const::Ty(_, c) = self - && let ty::ConstKind::Value(cv) = c.kind() - && cv.ty.is_primitive() - { - // Avoid the `valtree_to_const_val` query. Can only be done on primitive types that - // are valtree leaves, and *not* on references. (References should return the - // pointer here, which valtrees don't represent.) - Some(cv.valtree.unwrap_leaf().into()) + if let Const::Ty(_, c) = self { + // We don't evaluate anything for type system constants as normalizing + // the MIR will handle this for us + c.try_to_scalar() } else { self.eval(tcx, typing_env, DUMMY_SP).ok()?.try_to_scalar() } diff --git a/compiler/rustc_middle/src/mir/mod.rs b/compiler/rustc_middle/src/mir/mod.rs index 4ae4152cfb93..418cdea01660 100644 --- a/compiler/rustc_middle/src/mir/mod.rs +++ b/compiler/rustc_middle/src/mir/mod.rs @@ -620,6 +620,10 @@ impl<'tcx> Body<'tcx> { let bits = eval_mono_const(constant)?; return Some((bits, targets)); } + Operand::RuntimeChecks(check) => { + let bits = check.value(tcx.sess) as u128; + return Some((bits, targets)); + } Operand::Move(place) | Operand::Copy(place) => place, }; @@ -649,9 +653,6 @@ impl<'tcx> Body<'tcx> { } match rvalue { - Rvalue::NullaryOp(NullOp::RuntimeChecks(kind)) => { - Some((kind.value(tcx.sess) as u128, targets)) - } Rvalue::Use(Operand::Constant(constant)) => { let bits = eval_mono_const(constant)?; Some((bits, targets)) diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs index b225dd4ef60a..a31f03362a3d 100644 --- a/compiler/rustc_middle/src/mir/pretty.rs +++ b/compiler/rustc_middle/src/mir/pretty.rs @@ -1097,15 +1097,6 @@ impl<'tcx> Debug for Rvalue<'tcx> { BinaryOp(ref op, box (ref a, ref b)) => write!(fmt, "{op:?}({a:?}, {b:?})"), UnaryOp(ref op, ref a) => write!(fmt, "{op:?}({a:?})"), Discriminant(ref place) => write!(fmt, "discriminant({place:?})"), - NullaryOp(ref op) => match op { - NullOp::RuntimeChecks(RuntimeChecks::UbChecks) => write!(fmt, "UbChecks()"), - NullOp::RuntimeChecks(RuntimeChecks::ContractChecks) => { - write!(fmt, "ContractChecks()") - } - NullOp::RuntimeChecks(RuntimeChecks::OverflowChecks) => { - write!(fmt, "OverflowChecks()") - } - }, ThreadLocalRef(did) => ty::tls::with(|tcx| { let muta = tcx.static_mutability(did).unwrap().prefix_str(); write!(fmt, "&/*tls*/ {}{}", muta, tcx.def_path_str(did)) @@ -1264,6 +1255,7 @@ impl<'tcx> Debug for Operand<'tcx> { Constant(ref a) => write!(fmt, "{a:?}"), Copy(ref place) => write!(fmt, "copy {place:?}"), Move(ref place) => write!(fmt, "move {place:?}"), + RuntimeChecks(checks) => write!(fmt, "{checks:?}"), } } } @@ -1904,7 +1896,8 @@ fn pretty_print_const_value_tcx<'tcx>( // Aggregates, printed as array/tuple/struct/variant construction syntax. // // NB: the `has_non_region_param` check ensures that we can use - // the `destructure_const` query with an empty `ty::ParamEnv` without + // the `try_destructure_mir_constant_for_user_output ` query with + // an empty `TypingEnv::fully_monomorphized` without // introducing ICEs (e.g. via `layout_of`) from missing bounds. // E.g. `transmute([0usize; 2]): (u8, *mut T)` needs to know `T: Sized` // to be able to destructure the tuple into `(0u8, *mut T)` diff --git a/compiler/rustc_middle/src/mir/statement.rs b/compiler/rustc_middle/src/mir/statement.rs index bd4188dd0ff4..c1f8c46baddb 100644 --- a/compiler/rustc_middle/src/mir/statement.rs +++ b/compiler/rustc_middle/src/mir/statement.rs @@ -25,7 +25,7 @@ impl<'tcx> Statement<'tcx> { /// Changes a statement to a nop. This is both faster than deleting instructions and avoids /// invalidating statement indices in `Location`s. pub fn make_nop(&mut self, drop_debuginfo: bool) { - if matches!(self.kind, StatementKind::Nop) { + if self.kind == StatementKind::Nop { return; } let replaced_stmt = std::mem::replace(&mut self.kind, StatementKind::Nop); @@ -642,7 +642,7 @@ impl<'tcx> Operand<'tcx> { pub fn to_copy(&self) -> Self { match *self { - Operand::Copy(_) | Operand::Constant(_) => self.clone(), + Operand::Copy(_) | Operand::Constant(_) | Operand::RuntimeChecks(_) => self.clone(), Operand::Move(place) => Operand::Copy(place), } } @@ -652,7 +652,7 @@ impl<'tcx> Operand<'tcx> { pub fn place(&self) -> Option> { match self { Operand::Copy(place) | Operand::Move(place) => Some(*place), - Operand::Constant(_) => None, + Operand::Constant(_) | Operand::RuntimeChecks(_) => None, } } @@ -661,7 +661,7 @@ impl<'tcx> Operand<'tcx> { pub fn constant(&self) -> Option<&ConstOperand<'tcx>> { match self { Operand::Constant(x) => Some(&**x), - Operand::Copy(_) | Operand::Move(_) => None, + Operand::Copy(_) | Operand::Move(_) | Operand::RuntimeChecks(_) => None, } } @@ -681,6 +681,7 @@ impl<'tcx> Operand<'tcx> { match self { &Operand::Copy(ref l) | &Operand::Move(ref l) => l.ty(local_decls, tcx).ty, Operand::Constant(c) => c.const_.ty(), + Operand::RuntimeChecks(_) => tcx.types.bool, } } @@ -693,6 +694,8 @@ impl<'tcx> Operand<'tcx> { local_decls.local_decls()[l.local].source_info.span } Operand::Constant(c) => c.span, + // User code should not contain this operand, so we should not need this span. + Operand::RuntimeChecks(_) => DUMMY_SP, } } } @@ -756,7 +759,6 @@ impl<'tcx> Rvalue<'tcx> { _, ) | Rvalue::BinaryOp(_, _) - | Rvalue::NullaryOp(_) | Rvalue::UnaryOp(_, _) | Rvalue::Discriminant(_) | Rvalue::Aggregate(_, _) @@ -794,7 +796,6 @@ impl<'tcx> Rvalue<'tcx> { op.ty(tcx, arg_ty) } Rvalue::Discriminant(ref place) => place.ty(local_decls, tcx).ty.discriminant_ty(tcx), - Rvalue::NullaryOp(NullOp::RuntimeChecks(_)) => tcx.types.bool, Rvalue::Aggregate(ref ak, ref ops) => match **ak { AggregateKind::Array(ty) => Ty::new_array(tcx, ty, ops.len() as u64), AggregateKind::Tuple => { @@ -858,14 +859,6 @@ impl BorrowKind { } } -impl NullOp { - pub fn ty<'tcx>(&self, tcx: TyCtxt<'tcx>) -> Ty<'tcx> { - match self { - NullOp::RuntimeChecks(_) => tcx.types.bool, - } - } -} - impl<'tcx> UnOp { pub fn ty(&self, tcx: TyCtxt<'tcx>, arg_ty: Ty<'tcx>) -> Ty<'tcx> { match self { diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 204ad4815147..83e9a1f1784b 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -1327,6 +1327,10 @@ pub enum Operand<'tcx> { /// Constants are already semantically values, and remain unchanged. Constant(Box>), + + /// Query the compilation session of the current crate for a particular flag. This is not quite + /// a const since its value can differ across crates within a single crate graph. + RuntimeChecks(RuntimeChecks), } #[derive(Clone, Copy, PartialEq, TyEncodable, TyDecodable, Hash, HashStable)] @@ -1418,9 +1422,6 @@ pub enum Rvalue<'tcx> { /// matching types and return a value of that type. BinaryOp(BinOp, Box<(Operand<'tcx>, Operand<'tcx>)>), - /// Computes a value as described by the operation. - NullaryOp(NullOp), - /// Exactly like `BinaryOp`, but less operands. /// /// Also does two's-complement arithmetic. Negation requires a signed integer or a float; @@ -1561,12 +1562,6 @@ pub enum AggregateKind<'tcx> { RawPtr(Ty<'tcx>, Mutability), } -#[derive(Copy, Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)] -pub enum NullOp { - /// Returns whether we should perform some checking at runtime. - RuntimeChecks(RuntimeChecks), -} - #[derive(Copy, Clone, Debug, PartialEq, Eq, TyEncodable, TyDecodable, Hash, HashStable)] pub enum RuntimeChecks { /// Returns whether we should perform some UB-checking at runtime. diff --git a/compiler/rustc_middle/src/mir/traversal.rs b/compiler/rustc_middle/src/mir/traversal.rs index 7f6c7376501f..a3c0a5b83ffa 100644 --- a/compiler/rustc_middle/src/mir/traversal.rs +++ b/compiler/rustc_middle/src/mir/traversal.rs @@ -293,9 +293,9 @@ pub fn reverse_postorder<'a, 'tcx>( /// reachable. /// /// Such a traversal is mostly useful because it lets us skip lowering the `false` side -/// of `if ::CONST`, as well as [`NullOp::RuntimeChecks`]. +/// of `if ::CONST`, as well as [`Operand::RuntimeChecks`]. /// -/// [`NullOp::RuntimeChecks`]: rustc_middle::mir::NullOp::RuntimeChecks +/// [`Operand::RuntimeChecks`]: rustc_middle::mir::Operand::RuntimeChecks pub fn mono_reachable<'a, 'tcx>( body: &'a Body<'tcx>, tcx: TyCtxt<'tcx>, diff --git a/compiler/rustc_middle/src/mir/visit.rs b/compiler/rustc_middle/src/mir/visit.rs index 6d251988cbbd..07a36aef4320 100644 --- a/compiler/rustc_middle/src/mir/visit.rs +++ b/compiler/rustc_middle/src/mir/visit.rs @@ -775,8 +775,6 @@ macro_rules! make_mir_visitor { ); } - Rvalue::NullaryOp(_op) => {} - Rvalue::Aggregate(kind, operands) => { let kind = &$($mutability)? **kind; match kind { @@ -847,6 +845,7 @@ macro_rules! make_mir_visitor { Operand::Constant(constant) => { self.visit_const_operand(constant, location); } + Operand::RuntimeChecks(_) => {} } } @@ -972,10 +971,7 @@ macro_rules! make_mir_visitor { self.visit_span($(& $mutability)? *span); match const_ { Const::Ty(_, ct) => self.visit_ty_const($(&$mutability)? *ct, location), - Const::Val(_, ty) => { - self.visit_ty($(& $mutability)? *ty, TyContext::Location(location)); - } - Const::Unevaluated(_, ty) => { + Const::Val(_, ty) | Const::Unevaluated(_, ty) => { self.visit_ty($(& $mutability)? *ty, TyContext::Location(location)); } } diff --git a/compiler/rustc_middle/src/query/erase.rs b/compiler/rustc_middle/src/query/erase.rs index 774dd88997f2..711a597d4603 100644 --- a/compiler/rustc_middle/src/query/erase.rs +++ b/compiler/rustc_middle/src/query/erase.rs @@ -402,7 +402,7 @@ tcx_lifetime! { rustc_middle::ty::ClauseKind, rustc_middle::ty::ClosureTypeInfo, rustc_middle::ty::Const, - rustc_middle::ty::DestructuredConst, + rustc_middle::ty::DestructuredAdtConst, rustc_middle::ty::ExistentialTraitRef, rustc_middle::ty::FnSig, rustc_middle::ty::GenericArg, diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 676f0d82e4fb..b2473052b442 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -1408,12 +1408,6 @@ rustc_queries! { desc { "converting type-level constant value to MIR constant value"} } - /// Destructures array, ADT or tuple constants into the constants - /// of their fields. - query destructure_const(key: ty::Const<'tcx>) -> ty::DestructuredConst<'tcx> { - desc { "destructuring type level constant"} - } - // FIXME get rid of this with valtrees query lit_to_const( key: LitToConstInput<'tcx> diff --git a/compiler/rustc_middle/src/thir.rs b/compiler/rustc_middle/src/thir.rs index edbb736128cf..31745cae3c06 100644 --- a/compiler/rustc_middle/src/thir.rs +++ b/compiler/rustc_middle/src/thir.rs @@ -14,7 +14,7 @@ use std::ops::Index; use std::sync::Arc; use rustc_abi::{FieldIdx, Integer, Size, VariantIdx}; -use rustc_ast::{AsmMacro, InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_ast::{AsmMacro, InlineAsmOptions, InlineAsmTemplatePiece, Mutability}; use rustc_hir as hir; use rustc_hir::def_id::DefId; use rustc_hir::{BindingMode, ByRef, HirId, MatchSource, RangeEnd}; @@ -811,11 +811,11 @@ pub enum PatKind<'tcx> { DerefPattern { subpattern: Box>, /// Whether the pattern scrutinee needs to be borrowed in order to call `Deref::deref` or - /// `DerefMut::deref_mut`, and if so, which. This is `ByRef::No` for deref patterns on + /// `DerefMut::deref_mut`, and if so, which. This is `DerefPatBorrowMode::Box` for deref patterns on /// boxes; they are lowered using a built-in deref rather than a method call, thus they /// don't borrow the scrutinee. #[type_visitable(ignore)] - borrow: ByRef, + borrow: DerefPatBorrowMode, }, /// One of the following: @@ -879,6 +879,12 @@ pub enum PatKind<'tcx> { Error(ErrorGuaranteed), } +#[derive(Copy, Clone, Debug, HashStable)] +pub enum DerefPatBorrowMode { + Borrow(Mutability), + Box, +} + /// A range pattern. /// The boundaries must be of the same type and that type must be numeric. #[derive(Clone, Debug, PartialEq, HashStable, TypeVisitable)] @@ -922,7 +928,7 @@ impl<'tcx> PatRange<'tcx> { let lo_is_min = match self.lo { PatRangeBoundary::NegInfinity => true, PatRangeBoundary::Finite(value) => { - let lo = value.try_to_scalar_int().unwrap().to_bits(size) ^ bias; + let lo = value.to_leaf().to_bits(size) ^ bias; lo <= min } PatRangeBoundary::PosInfinity => false, @@ -931,7 +937,7 @@ impl<'tcx> PatRange<'tcx> { let hi_is_max = match self.hi { PatRangeBoundary::NegInfinity => false, PatRangeBoundary::Finite(value) => { - let hi = value.try_to_scalar_int().unwrap().to_bits(size) ^ bias; + let hi = value.to_leaf().to_bits(size) ^ bias; hi > max || hi == max && self.end == RangeEnd::Included } PatRangeBoundary::PosInfinity => true, @@ -1023,7 +1029,7 @@ impl<'tcx> PatRangeBoundary<'tcx> { } pub fn to_bits(self, ty: Ty<'tcx>, tcx: TyCtxt<'tcx>) -> u128 { match self { - Self::Finite(value) => value.try_to_scalar_int().unwrap().to_bits_unchecked(), + Self::Finite(value) => value.to_leaf().to_bits_unchecked(), Self::NegInfinity => { // Unwrap is ok because the type is known to be numeric. ty.numeric_min_and_max_as_bits(tcx).unwrap().0 @@ -1051,7 +1057,7 @@ impl<'tcx> PatRangeBoundary<'tcx> { // many ranges such as '\u{037A}'..='\u{037F}', and chars can be compared // in this way. (Finite(a), Finite(b)) if matches!(ty.kind(), ty::Int(_) | ty::Uint(_) | ty::Char) => { - if let (Some(a), Some(b)) = (a.try_to_scalar_int(), b.try_to_scalar_int()) { + if let (Some(a), Some(b)) = (a.try_to_leaf(), b.try_to_leaf()) { let sz = ty.primitive_size(tcx); let cmp = match ty.kind() { ty::Uint(_) | ty::Char => a.to_uint(sz).cmp(&b.to_uint(sz)), diff --git a/compiler/rustc_middle/src/ty/consts.rs b/compiler/rustc_middle/src/ty/consts.rs index 787ea5f9363d..da3caf0bb210 100644 --- a/compiler/rustc_middle/src/ty/consts.rs +++ b/compiler/rustc_middle/src/ty/consts.rs @@ -6,6 +6,7 @@ use rustc_macros::{HashStable, TyDecodable, TyEncodable}; use rustc_type_ir::walk::TypeWalker; use rustc_type_ir::{self as ir, TypeFlags, WithCachedTypeInfo}; +use crate::mir::interpret::Scalar; use crate::ty::{self, Ty, TyCtxt}; mod int; @@ -260,7 +261,7 @@ impl<'tcx> Const<'tcx> { /// Attempts to convert to a value. /// - /// Note that this does not evaluate the constant. + /// Note that this does not normalize the constant. pub fn try_to_value(self) -> Option> { match self.kind() { ty::ConstKind::Value(cv) => Some(cv), @@ -268,6 +269,45 @@ impl<'tcx> Const<'tcx> { } } + /// Converts to a `ValTreeKind::Leaf` value, `panic`'ing + /// if this constant is some other kind. + /// + /// Note that this does not normalize the constant. + #[inline] + pub fn to_leaf(self) -> ScalarInt { + self.to_value().to_leaf() + } + + /// Converts to a `ValTreeKind::Branch` value, `panic`'ing + /// if this constant is some other kind. + /// + /// Note that this does not normalize the constant. + #[inline] + pub fn to_branch(self) -> &'tcx [ty::Const<'tcx>] { + self.to_value().to_branch() + } + + /// Attempts to convert to a `ValTreeKind::Leaf` value. + /// + /// Note that this does not normalize the constant. + pub fn try_to_leaf(self) -> Option { + self.try_to_value()?.try_to_leaf() + } + + /// Attempts to convert to a `ValTreeKind::Leaf` value. + /// + /// Note that this does not normalize the constant. + pub fn try_to_scalar(self) -> Option { + self.try_to_leaf().map(Scalar::Int) + } + + /// Attempts to convert to a `ValTreeKind::Branch` value. + /// + /// Note that this does not normalize the constant. + pub fn try_to_branch(self) -> Option<&'tcx [ty::Const<'tcx>]> { + self.try_to_value()?.try_to_branch() + } + /// Convenience method to extract the value of a usize constant, /// useful to get the length of an array type. /// diff --git a/compiler/rustc_middle/src/ty/consts/valtree.rs b/compiler/rustc_middle/src/ty/consts/valtree.rs index a14e47d70821..6501ddeed6fa 100644 --- a/compiler/rustc_middle/src/ty/consts/valtree.rs +++ b/compiler/rustc_middle/src/ty/consts/valtree.rs @@ -1,91 +1,41 @@ use std::fmt; use std::ops::Deref; +use rustc_abi::{FIRST_VARIANT, VariantIdx}; use rustc_data_structures::intern::Interned; use rustc_hir::def::Namespace; -use rustc_macros::{HashStable, Lift, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable}; +use rustc_macros::{ + HashStable, Lift, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable, extension, +}; use super::ScalarInt; use crate::mir::interpret::{ErrorHandled, Scalar}; use crate::ty::print::{FmtPrinter, PrettyPrinter}; -use crate::ty::{self, Ty, TyCtxt}; +use crate::ty::{self, Ty, TyCtxt, ValTreeKind}; -/// This datastructure is used to represent the value of constants used in the type system. -/// -/// We explicitly choose a different datastructure from the way values are processed within -/// CTFE, as in the type system equal values (according to their `PartialEq`) must also have -/// equal representation (`==` on the rustc data structure, e.g. `ValTree`) and vice versa. -/// Since CTFE uses `AllocId` to represent pointers, it often happens that two different -/// `AllocId`s point to equal values. So we may end up with different representations for -/// two constants whose value is `&42`. Furthermore any kind of struct that has padding will -/// have arbitrary values within that padding, even if the values of the struct are the same. -/// -/// `ValTree` does not have this problem with representation, as it only contains integers or -/// lists of (nested) `ValTree`. -#[derive(Clone, Debug, Hash, Eq, PartialEq)] -#[derive(HashStable, TyEncodable, TyDecodable)] -pub enum ValTreeKind<'tcx> { - /// integers, `bool`, `char` are represented as scalars. - /// See the `ScalarInt` documentation for how `ScalarInt` guarantees that equal values - /// of these types have the same representation. - Leaf(ScalarInt), - - //SliceOrStr(ValSlice<'tcx>), - // don't use SliceOrStr for now - /// The fields of any kind of aggregate. Structs, tuples and arrays are represented by - /// listing their fields' values in order. - /// - /// Enums are represented by storing their variant index as a u32 field, followed by all - /// the fields of the variant. - /// - /// ZST types are represented as an empty slice. - Branch(Box<[ValTree<'tcx>]>), -} - -impl<'tcx> ValTreeKind<'tcx> { - #[inline] - pub fn unwrap_leaf(&self) -> ScalarInt { - match self { - Self::Leaf(s) => *s, - _ => bug!("expected leaf, got {:?}", self), - } - } - - #[inline] - pub fn unwrap_branch(&self) -> &[ValTree<'tcx>] { - match self { - Self::Branch(branch) => &**branch, - _ => bug!("expected branch, got {:?}", self), - } - } - - pub fn try_to_scalar(&self) -> Option { - self.try_to_scalar_int().map(Scalar::Int) - } - - pub fn try_to_scalar_int(&self) -> Option { - match self { - Self::Leaf(s) => Some(*s), - Self::Branch(_) => None, - } - } - - pub fn try_to_branch(&self) -> Option<&[ValTree<'tcx>]> { - match self { - Self::Branch(branch) => Some(&**branch), - Self::Leaf(_) => None, - } +#[extension(pub trait ValTreeKindExt<'tcx>)] +impl<'tcx> ty::ValTreeKind> { + fn try_to_scalar(&self) -> Option { + self.try_to_leaf().map(Scalar::Int) } } /// An interned valtree. Use this rather than `ValTreeKind`, whenever possible. /// -/// See the docs of [`ValTreeKind`] or the [dev guide] for an explanation of this type. +/// See the docs of [`ty::ValTreeKind`] or the [dev guide] for an explanation of this type. /// /// [dev guide]: https://rustc-dev-guide.rust-lang.org/mir/index.html#valtrees #[derive(Copy, Clone, Hash, Eq, PartialEq)] #[derive(HashStable)] -pub struct ValTree<'tcx>(pub(crate) Interned<'tcx, ValTreeKind<'tcx>>); +// FIXME(mgca): Try not interning here. We already intern `ty::Const` which `ValTreeKind` +// recurses through +pub struct ValTree<'tcx>(pub(crate) Interned<'tcx, ty::ValTreeKind>>); + +impl<'tcx> rustc_type_ir::inherent::ValTree> for ValTree<'tcx> { + fn kind(&self) -> &ty::ValTreeKind> { + &self + } +} impl<'tcx> ValTree<'tcx> { /// Returns the zero-sized valtree: `Branch([])`. @@ -94,28 +44,33 @@ impl<'tcx> ValTree<'tcx> { } pub fn is_zst(self) -> bool { - matches!(*self, ValTreeKind::Branch(box [])) + matches!(*self, ty::ValTreeKind::Branch(box [])) } pub fn from_raw_bytes(tcx: TyCtxt<'tcx>, bytes: &[u8]) -> Self { - let branches = bytes.iter().map(|&b| Self::from_scalar_int(tcx, b.into())); + let branches = bytes.iter().map(|&b| { + ty::Const::new_value(tcx, Self::from_scalar_int(tcx, b.into()), tcx.types.u8) + }); Self::from_branches(tcx, branches) } - pub fn from_branches(tcx: TyCtxt<'tcx>, branches: impl IntoIterator) -> Self { - tcx.intern_valtree(ValTreeKind::Branch(branches.into_iter().collect())) + pub fn from_branches( + tcx: TyCtxt<'tcx>, + branches: impl IntoIterator>, + ) -> Self { + tcx.intern_valtree(ty::ValTreeKind::Branch(branches.into_iter().collect())) } pub fn from_scalar_int(tcx: TyCtxt<'tcx>, i: ScalarInt) -> Self { - tcx.intern_valtree(ValTreeKind::Leaf(i)) + tcx.intern_valtree(ty::ValTreeKind::Leaf(i)) } } impl<'tcx> Deref for ValTree<'tcx> { - type Target = &'tcx ValTreeKind<'tcx>; + type Target = &'tcx ty::ValTreeKind>; #[inline] - fn deref(&self) -> &&'tcx ValTreeKind<'tcx> { + fn deref(&self) -> &&'tcx ty::ValTreeKind> { &self.0.0 } } @@ -154,7 +109,7 @@ impl<'tcx> Value<'tcx> { let (ty::Bool | ty::Char | ty::Uint(_) | ty::Int(_) | ty::Float(_)) = self.ty.kind() else { return None; }; - let scalar = self.valtree.try_to_scalar_int()?; + let scalar = self.try_to_leaf()?; let input = typing_env.with_post_analysis_normalized(tcx).as_query_input(self.ty); let size = tcx.layout_of(input).ok()?.size; Some(scalar.to_bits(size)) @@ -164,14 +119,14 @@ impl<'tcx> Value<'tcx> { if !self.ty.is_bool() { return None; } - self.valtree.try_to_scalar_int()?.try_to_bool().ok() + self.try_to_leaf()?.try_to_bool().ok() } pub fn try_to_target_usize(self, tcx: TyCtxt<'tcx>) -> Option { if !self.ty.is_usize() { return None; } - self.valtree.try_to_scalar_int().map(|s| s.to_target_usize(tcx)) + self.try_to_leaf().map(|s| s.to_target_usize(tcx)) } /// Get the values inside the ValTree as a slice of bytes. This only works for @@ -192,9 +147,68 @@ impl<'tcx> Value<'tcx> { _ => return None, } - Some(tcx.arena.alloc_from_iter( - self.valtree.unwrap_branch().into_iter().map(|v| v.unwrap_leaf().to_u8()), - )) + Some(tcx.arena.alloc_from_iter(self.to_branch().into_iter().map(|ct| ct.to_leaf().to_u8()))) + } + + /// Converts to a `ValTreeKind::Leaf` value, `panic`'ing + /// if this constant is some other kind. + #[inline] + pub fn to_leaf(self) -> ScalarInt { + match &**self.valtree { + ValTreeKind::Leaf(s) => *s, + ValTreeKind::Branch(..) => bug!("expected leaf, got {:?}", self), + } + } + + /// Converts to a `ValTreeKind::Branch` value, `panic`'ing + /// if this constant is some other kind. + #[inline] + pub fn to_branch(self) -> &'tcx [ty::Const<'tcx>] { + match &**self.valtree { + ValTreeKind::Branch(branch) => &**branch, + ValTreeKind::Leaf(..) => bug!("expected branch, got {:?}", self), + } + } + + /// Attempts to convert to a `ValTreeKind::Leaf` value. + pub fn try_to_leaf(self) -> Option { + match &**self.valtree { + ValTreeKind::Leaf(s) => Some(*s), + ValTreeKind::Branch(_) => None, + } + } + + /// Attempts to convert to a `ValTreeKind::Leaf` value. + pub fn try_to_scalar(&self) -> Option { + self.try_to_leaf().map(Scalar::Int) + } + + /// Attempts to convert to a `ValTreeKind::Branch` value. + pub fn try_to_branch(self) -> Option<&'tcx [ty::Const<'tcx>]> { + match &**self.valtree { + ValTreeKind::Branch(branch) => Some(&**branch), + ValTreeKind::Leaf(_) => None, + } + } + + /// Destructures array, ADT or tuple constants into the constants + /// of their fields. + pub fn destructure_adt_const(&self) -> ty::DestructuredAdtConst<'tcx> { + let fields = self.to_branch(); + + let (variant, fields) = match self.ty.kind() { + ty::Adt(def, _) if def.variants().is_empty() => { + bug!("unreachable") + } + ty::Adt(def, _) if def.is_enum() => { + let (head, rest) = fields.split_first().unwrap(); + (VariantIdx::from_u32(head.to_leaf().to_u32()), rest) + } + ty::Adt(_, _) => (FIRST_VARIANT, fields), + _ => bug!("destructure_adt_const called on non-ADT type: {:?}", self.ty), + }; + + ty::DestructuredAdtConst { variant, fields } } } diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 471bd1d937e9..25cc739f5ff9 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -165,6 +165,7 @@ impl<'tcx> Interner for TyCtxt<'tcx> { type ValueConst = ty::Value<'tcx>; type ExprConst = ty::Expr<'tcx>; type ValTree = ty::ValTree<'tcx>; + type ScalarInt = ty::ScalarInt; type Region = Region<'tcx>; type EarlyParamRegion = ty::EarlyParamRegion; @@ -954,7 +955,7 @@ pub struct CtxtInterners<'tcx> { fields: InternedSet<'tcx, List>, local_def_ids: InternedSet<'tcx, List>, captures: InternedSet<'tcx, List<&'tcx ty::CapturedPlace<'tcx>>>, - valtree: InternedSet<'tcx, ty::ValTreeKind<'tcx>>, + valtree: InternedSet<'tcx, ty::ValTreeKind>>, patterns: InternedSet<'tcx, List>>, outlives: InternedSet<'tcx, List>>, } @@ -2272,6 +2273,12 @@ impl<'tcx> TyCtxt<'tcx> { #[inline] pub fn local_crate_exports_generics(self) -> bool { + // compiler-builtins has some special treatment in codegen, which can result in confusing + // behavior if another crate ends up calling into its monomorphizations. + // https://github.com/rust-lang/rust/issues/150173 + if self.is_compiler_builtins(LOCAL_CRATE) { + return false; + } self.crate_types().iter().any(|crate_type| { match crate_type { CrateType::Executable @@ -2654,7 +2661,7 @@ struct InternedInSet<'tcx, T: ?Sized + PointeeSized>(&'tcx T); impl<'tcx, T: 'tcx + ?Sized + PointeeSized> Clone for InternedInSet<'tcx, T> { fn clone(&self) -> Self { - InternedInSet(self.0) + *self } } @@ -2777,7 +2784,7 @@ macro_rules! direct_interners { // crate only, and have a corresponding `mk_` function. direct_interners! { region: pub(crate) intern_region(RegionKind<'tcx>): Region -> Region<'tcx>, - valtree: pub(crate) intern_valtree(ValTreeKind<'tcx>): ValTree -> ValTree<'tcx>, + valtree: pub(crate) intern_valtree(ValTreeKind>): ValTree -> ValTree<'tcx>, pat: pub mk_pat(PatternKind<'tcx>): Pattern -> Pattern<'tcx>, const_allocation: pub mk_const_alloc(Allocation): ConstAllocation -> ConstAllocation<'tcx>, layout: pub mk_layout(LayoutData): Layout -> Layout<'tcx>, diff --git a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs index 953ad62be0a8..d03e593e37b9 100644 --- a/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs +++ b/compiler/rustc_middle/src/ty/inhabitedness/inhabited_predicate.rs @@ -160,7 +160,7 @@ impl<'tcx> InhabitedPredicate<'tcx> { pub fn all(tcx: TyCtxt<'tcx>, iter: impl IntoIterator) -> Self { let mut result = Self::True; for pred in iter { - if matches!(pred, Self::False) { + if pred == Self::False { return Self::False; } result = result.and(tcx, pred); @@ -171,7 +171,7 @@ impl<'tcx> InhabitedPredicate<'tcx> { pub fn any(tcx: TyCtxt<'tcx>, iter: impl IntoIterator) -> Self { let mut result = Self::False; for pred in iter { - if matches!(pred, Self::True) { + if pred == Self::True { return Self::True; } result = result.or(tcx, pred); diff --git a/compiler/rustc_middle/src/ty/mod.rs b/compiler/rustc_middle/src/ty/mod.rs index 91e6e6326c09..e2a94b607de1 100644 --- a/compiler/rustc_middle/src/ty/mod.rs +++ b/compiler/rustc_middle/src/ty/mod.rs @@ -31,7 +31,7 @@ use rustc_ast::AttrVec; use rustc_ast::expand::typetree::{FncTree, Kind, Type, TypeTree}; use rustc_ast::node_id::NodeMap; pub use rustc_ast_ir::{Movability, Mutability, try_visit}; -use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet}; +use rustc_data_structures::fx::{FxHashSet, FxIndexMap, FxIndexSet}; use rustc_data_structures::intern::Interned; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::steal::Steal; @@ -77,7 +77,7 @@ pub use self::closure::{ }; pub use self::consts::{ AnonConstKind, AtomicOrdering, Const, ConstInt, ConstKind, ConstToValTreeResult, Expr, - ExprKind, ScalarInt, SimdAlign, UnevaluatedConst, ValTree, ValTreeKind, Value, + ExprKind, ScalarInt, SimdAlign, UnevaluatedConst, ValTree, ValTreeKindExt, Value, }; pub use self::context::{ CtxtInterners, CurrentGcx, Feed, FreeRegionInfo, GlobalCtxt, Lift, TyCtxt, TyCtxtFeed, tls, @@ -196,8 +196,6 @@ pub struct ResolverGlobalCtxt { /// This struct is meant to be consumed by lowering. #[derive(Debug)] pub struct ResolverAstLowering { - pub legacy_const_generic_args: FxHashMap>>, - /// Resolutions for nodes that have a single resolution. pub partial_res_map: NodeMap, /// Resolutions for import nodes, which have multiple resolutions in different namespaces. @@ -222,6 +220,8 @@ pub struct ResolverAstLowering { /// Information about functions signatures for delegation items expansion pub delegation_fn_sigs: LocalDefIdMap, + // Information about delegations which is used when handling recursive delegations + pub delegation_infos: LocalDefIdMap, } bitflags::bitflags! { @@ -234,14 +234,27 @@ bitflags::bitflags! { pub const DELEGATION_INHERIT_ATTRS_START: DelegationFnSigAttrs = DelegationFnSigAttrs::MUST_USE; +#[derive(Debug)] +pub struct DelegationInfo { + // NodeId (either delegation.id or item_id in case of a trait impl) for signature resolution, + // for details see https://github.com/rust-lang/rust/issues/118212#issuecomment-2160686914 + pub resolution_node: ast::NodeId, + pub attrs: DelegationAttrs, +} + +#[derive(Debug)] +pub struct DelegationAttrs { + pub flags: DelegationFnSigAttrs, + pub to_inherit: AttrVec, +} + #[derive(Debug)] pub struct DelegationFnSig { pub header: ast::FnHeader, pub param_count: usize, pub has_self: bool, pub c_variadic: bool, - pub attrs_flags: DelegationFnSigAttrs, - pub to_inherit_attrs: AttrVec, + pub attrs: DelegationAttrs, } #[derive(Clone, Copy, Debug, HashStable)] @@ -2318,8 +2331,8 @@ impl<'tcx> fmt::Debug for SymbolName<'tcx> { /// The constituent parts of a type level constant of kind ADT or array. #[derive(Copy, Clone, Debug, HashStable)] -pub struct DestructuredConst<'tcx> { - pub variant: Option, +pub struct DestructuredAdtConst<'tcx> { + pub variant: VariantIdx, pub fields: &'tcx [ty::Const<'tcx>], } diff --git a/compiler/rustc_middle/src/ty/pattern.rs b/compiler/rustc_middle/src/ty/pattern.rs index 335e5c064743..6acf0aff800f 100644 --- a/compiler/rustc_middle/src/ty/pattern.rs +++ b/compiler/rustc_middle/src/ty/pattern.rs @@ -72,7 +72,7 @@ impl<'tcx> IrPrint> for TyCtxt<'tcx> { write!(f, "{start}")?; if let Some(c) = end.try_to_value() { - let end = c.valtree.unwrap_leaf(); + let end = c.to_leaf(); let size = end.size(); let max = match c.ty.kind() { ty::Int(_) => { diff --git a/compiler/rustc_middle/src/ty/print/pretty.rs b/compiler/rustc_middle/src/ty/print/pretty.rs index c4f2917ad9e5..2a65517de403 100644 --- a/compiler/rustc_middle/src/ty/print/pretty.rs +++ b/compiler/rustc_middle/src/ty/print/pretty.rs @@ -1919,66 +1919,65 @@ pub trait PrettyPrinter<'tcx>: Printer<'tcx> + fmt::Write { self.pretty_print_byte_str(bytes)?; return Ok(()); } - // Aggregates, printed as array/tuple/struct/variant construction syntax. - (ty::ValTreeKind::Branch(_), ty::Array(..) | ty::Tuple(..) | ty::Adt(..)) => { - let contents = self.tcx().destructure_const(ty::Const::new_value( - self.tcx(), - cv.valtree, - cv.ty, - )); - let fields = contents.fields.iter().copied(); + (ty::ValTreeKind::Branch(fields), ty::Array(..) | ty::Tuple(..)) => { + let fields_iter = fields.iter().copied(); + match *cv.ty.kind() { ty::Array(..) => { write!(self, "[")?; - self.comma_sep(fields)?; + self.comma_sep(fields_iter)?; write!(self, "]")?; } ty::Tuple(..) => { write!(self, "(")?; - self.comma_sep(fields)?; - if contents.fields.len() == 1 { + self.comma_sep(fields_iter)?; + if fields.len() == 1 { write!(self, ",")?; } write!(self, ")")?; } - ty::Adt(def, _) if def.variants().is_empty() => { - self.typed_value( - |this| { - write!(this, "unreachable()")?; - Ok(()) - }, - |this| this.print_type(cv.ty), - ": ", - )?; - } - ty::Adt(def, args) => { - let variant_idx = - contents.variant.expect("destructed const of adt without variant idx"); - let variant_def = &def.variant(variant_idx); - self.pretty_print_value_path(variant_def.def_id, args)?; - match variant_def.ctor_kind() { - Some(CtorKind::Const) => {} - Some(CtorKind::Fn) => { - write!(self, "(")?; - self.comma_sep(fields)?; - write!(self, ")")?; - } - None => { - write!(self, " {{ ")?; - let mut first = true; - for (field_def, field) in iter::zip(&variant_def.fields, fields) { - if !first { - write!(self, ", ")?; - } - write!(self, "{}: ", field_def.name)?; - field.print(self)?; - first = false; + _ => unreachable!(), + } + return Ok(()); + } + (ty::ValTreeKind::Branch(_), ty::Adt(def, args)) => { + let contents = cv.destructure_adt_const(); + let fields = contents.fields.iter().copied(); + + if def.variants().is_empty() { + self.typed_value( + |this| { + write!(this, "unreachable()")?; + Ok(()) + }, + |this| this.print_type(cv.ty), + ": ", + )?; + } else { + let variant_idx = contents.variant; + let variant_def = &def.variant(variant_idx); + self.pretty_print_value_path(variant_def.def_id, args)?; + match variant_def.ctor_kind() { + Some(CtorKind::Const) => {} + Some(CtorKind::Fn) => { + write!(self, "(")?; + self.comma_sep(fields)?; + write!(self, ")")?; + } + None => { + write!(self, " {{ ")?; + let mut first = true; + for (field_def, field) in iter::zip(&variant_def.fields, fields) { + if !first { + write!(self, ", ")?; } - write!(self, " }}")?; + write!(self, "{}: ", field_def.name)?; + field.print(self)?; + first = false; } + write!(self, " }}")?; } } - _ => unreachable!(), } return Ok(()); } @@ -3381,7 +3380,7 @@ define_print_and_forward_display! { fn for_each_def(tcx: TyCtxt<'_>, mut collect_fn: impl for<'b> FnMut(&'b Ident, Namespace, DefId)) { // Iterate all (non-anonymous) local crate items no matter where they are defined. for id in tcx.hir_free_items() { - if matches!(tcx.def_kind(id.owner_id), DefKind::Use) { + if tcx.def_kind(id.owner_id) == DefKind::Use { continue; } diff --git a/compiler/rustc_middle/src/ty/structural_impls.rs b/compiler/rustc_middle/src/ty/structural_impls.rs index 4f70830002ce..1a5a3f3965fa 100644 --- a/compiler/rustc_middle/src/ty/structural_impls.rs +++ b/compiler/rustc_middle/src/ty/structural_impls.rs @@ -243,7 +243,6 @@ TrivialTypeTraversalImpls! { crate::mir::FakeReadCause, crate::mir::Local, crate::mir::MirPhase, - crate::mir::NullOp, crate::mir::Promoted, crate::mir::RawPtrKind, crate::mir::RetagKind, @@ -257,8 +256,8 @@ TrivialTypeTraversalImpls! { crate::ty::AssocItem, crate::ty::AssocKind, crate::ty::BoundRegion, + crate::ty::ScalarInt, crate::ty::UserTypeAnnotationIndex, - crate::ty::ValTree<'tcx>, crate::ty::abstract_const::NotConstEvaluatable, crate::ty::adjustment::AutoBorrowMutability, crate::ty::adjustment::PointerCoercion, @@ -284,6 +283,7 @@ TrivialTypeTraversalImpls! { // interners). TrivialTypeTraversalAndLiftImpls! { // tidy-alphabetical-start + crate::mir::RuntimeChecks, crate::ty::BoundTy, crate::ty::ParamTy, crate::ty::instance::ReifyReason, @@ -697,6 +697,37 @@ impl<'tcx> TypeSuperVisitable> for ty::Const<'tcx> { } } +impl<'tcx> TypeVisitable> for ty::ValTree<'tcx> { + fn visit_with>>(&self, visitor: &mut V) -> V::Result { + let inner: &ty::ValTreeKind> = &*self; + inner.visit_with(visitor) + } +} + +impl<'tcx> TypeFoldable> for ty::ValTree<'tcx> { + fn try_fold_with>>( + self, + folder: &mut F, + ) -> Result { + let inner: &ty::ValTreeKind> = &*self; + let new_inner = inner.clone().try_fold_with(folder)?; + + if inner == &new_inner { + Ok(self) + } else { + let valtree = folder.cx().intern_valtree(new_inner); + Ok(valtree) + } + } + + fn fold_with>>(self, folder: &mut F) -> Self { + let inner: &ty::ValTreeKind> = &*self; + let new_inner = inner.clone().fold_with(folder); + + if inner == &new_inner { self } else { folder.cx().intern_valtree(new_inner) } + } +} + impl<'tcx> TypeVisitable> for rustc_span::ErrorGuaranteed { fn visit_with>>(&self, visitor: &mut V) -> V::Result { visitor.visit_error(*self) diff --git a/compiler/rustc_middle/src/ty/typeck_results.rs b/compiler/rustc_middle/src/ty/typeck_results.rs index a3353661d9dd..2c5e4957d0ae 100644 --- a/compiler/rustc_middle/src/ty/typeck_results.rs +++ b/compiler/rustc_middle/src/ty/typeck_results.rs @@ -11,7 +11,6 @@ use rustc_hir::def_id::{DefId, LocalDefId, LocalDefIdMap}; use rustc_hir::hir_id::OwnerId; use rustc_hir::{ self as hir, BindingMode, ByRef, HirId, ItemLocalId, ItemLocalMap, ItemLocalSet, Mutability, - Pinnedness, }; use rustc_index::IndexVec; use rustc_macros::{HashStable, TyDecodable, TyEncodable, TypeFoldable, TypeVisitable}; @@ -21,6 +20,7 @@ use rustc_span::Span; use crate::hir::place::Place as HirPlace; use crate::infer::canonical::Canonical; use crate::mir::FakeReadCause; +use crate::thir::DerefPatBorrowMode; use crate::traits::ObligationCause; use crate::ty::{ self, BoundVar, CanonicalPolyFnSig, ClosureSizeProfileData, GenericArgKind, GenericArgs, @@ -491,13 +491,18 @@ impl<'tcx> TypeckResults<'tcx> { /// In most cases, if the pattern recursively contains a `ref mut` binding, we find the inner /// pattern's scrutinee by calling `DerefMut::deref_mut`, and otherwise we call `Deref::deref`. /// However, for boxes we can use a built-in deref instead, which doesn't borrow the scrutinee; - /// in this case, we return `ByRef::No`. - pub fn deref_pat_borrow_mode(&self, pointer_ty: Ty<'_>, inner: &hir::Pat<'_>) -> ByRef { + /// in this case, we return `DerefPatBorrowMode::Box`. + pub fn deref_pat_borrow_mode( + &self, + pointer_ty: Ty<'_>, + inner: &hir::Pat<'_>, + ) -> DerefPatBorrowMode { if pointer_ty.is_box() { - ByRef::No + DerefPatBorrowMode::Box } else { - let mutable = self.pat_has_ref_mut_binding(inner); - ByRef::Yes(Pinnedness::Not, if mutable { Mutability::Mut } else { Mutability::Not }) + let mutability = + if self.pat_has_ref_mut_binding(inner) { Mutability::Mut } else { Mutability::Not }; + DerefPatBorrowMode::Borrow(mutability) } } diff --git a/compiler/rustc_middle/src/ty/util.rs b/compiler/rustc_middle/src/ty/util.rs index fc03ad52b4bf..2797f2fcdb72 100644 --- a/compiler/rustc_middle/src/ty/util.rs +++ b/compiler/rustc_middle/src/ty/util.rs @@ -1394,8 +1394,10 @@ impl<'tcx> Ty<'tcx> { // This doesn't depend on regions, so try to minimize distinct // query keys used. - let erased = tcx.normalize_erasing_regions(typing_env, query_ty); - tcx.has_significant_drop_raw(typing_env.as_query_input(erased)) + // FIX: Use try_normalize to avoid crashing. If it fails, return true. + tcx.try_normalize_erasing_regions(typing_env, query_ty) + .map(|erased| tcx.has_significant_drop_raw(typing_env.as_query_input(erased))) + .unwrap_or(true) } } } diff --git a/compiler/rustc_mir_build/src/builder/custom/parse.rs b/compiler/rustc_mir_build/src/builder/custom/parse.rs index 10154461c339..c6ef362c6ea5 100644 --- a/compiler/rustc_mir_build/src/builder/custom/parse.rs +++ b/compiler/rustc_mir_build/src/builder/custom/parse.rs @@ -261,6 +261,7 @@ impl<'a, 'tcx> ParseCtxt<'a, 'tcx> { let value = match operand { Operand::Constant(c) => VarDebugInfoContents::Const(*c), Operand::Copy(p) | Operand::Move(p) => VarDebugInfoContents::Place(p), + Operand::RuntimeChecks(_) => unreachable!(), }; let dbginfo = VarDebugInfo { name, diff --git a/compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs b/compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs index b221318bf0b1..ddaa61c6cc91 100644 --- a/compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs +++ b/compiler/rustc_mir_build/src/builder/custom/parse/instruction.rs @@ -157,7 +157,7 @@ impl<'a, 'tcx> ParseCtxt<'a, 'tcx> { }); } }; - values.push(value.valtree.unwrap_leaf().to_bits_unchecked()); + values.push(value.to_leaf().to_bits_unchecked()); targets.push(self.parse_block(arm.body)?); } diff --git a/compiler/rustc_mir_build/src/builder/matches/buckets.rs b/compiler/rustc_mir_build/src/builder/matches/buckets.rs index f53e73d406c3..f8af50ee52fe 100644 --- a/compiler/rustc_mir_build/src/builder/matches/buckets.rs +++ b/compiler/rustc_mir_build/src/builder/matches/buckets.rs @@ -6,8 +6,7 @@ use rustc_middle::span_bug; use tracing::debug; use crate::builder::Builder; -use crate::builder::matches::test::is_switch_ty; -use crate::builder::matches::{Candidate, Test, TestBranch, TestCase, TestKind}; +use crate::builder::matches::{Candidate, PatConstKind, Test, TestBranch, TestKind, TestableCase}; /// Output of [`Builder::partition_candidates_into_buckets`]. pub(crate) struct PartitionedCandidates<'tcx, 'b, 'c> { @@ -140,12 +139,12 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // branch, so it can be removed. If false, the match pair is _compatible_ // with its test branch, but still needs a more specific test. let fully_matched; - let ret = match (&test.kind, &match_pair.test_case) { + let ret = match (&test.kind, &match_pair.testable_case) { // If we are performing a variant switch, then this // informs variant patterns, but nothing else. ( &TestKind::Switch { adt_def: tested_adt_def }, - &TestCase::Variant { adt_def, variant_index }, + &TestableCase::Variant { adt_def, variant_index }, ) => { assert_eq!(adt_def, tested_adt_def); fully_matched = true; @@ -157,26 +156,24 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // // FIXME(#29623) we could use PatKind::Range to rule // things out here, in some cases. - // - // FIXME(Zalathar): Is the `is_switch_ty` test unnecessary? - (TestKind::SwitchInt, &TestCase::Constant { value }) - if is_switch_ty(match_pair.pattern_ty) => - { + ( + TestKind::SwitchInt, + &TestableCase::Constant { value, kind: PatConstKind::IntOrChar }, + ) => { // An important invariant of candidate bucketing is that a candidate // must not match in multiple branches. For `SwitchInt` tests, adding // a new value might invalidate that property for range patterns that // have already been partitioned into the failure arm, so we must take care // not to add such values here. - let is_covering_range = |test_case: &TestCase<'tcx>| { - test_case.as_range().is_some_and(|range| { + let is_covering_range = |testable_case: &TestableCase<'tcx>| { + testable_case.as_range().is_some_and(|range| { matches!(range.contains(value, self.tcx), None | Some(true)) }) }; let is_conflicting_candidate = |candidate: &&mut Candidate<'tcx>| { - candidate - .match_pairs - .iter() - .any(|mp| mp.place == Some(test_place) && is_covering_range(&mp.test_case)) + candidate.match_pairs.iter().any(|mp| { + mp.place == Some(test_place) && is_covering_range(&mp.testable_case) + }) }; if prior_candidates .get(&TestBranch::Failure) @@ -189,7 +186,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { Some(TestBranch::Constant(value)) } } - (TestKind::SwitchInt, TestCase::Range(range)) => { + (TestKind::SwitchInt, TestableCase::Range(range)) => { // When performing a `SwitchInt` test, a range pattern can be // sorted into the failure arm if it doesn't contain _any_ of // the values being tested. (This restricts what values can be @@ -207,7 +204,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { }) } - (TestKind::If, TestCase::Constant { value }) => { + (TestKind::If, TestableCase::Constant { value, kind: PatConstKind::Bool }) => { fully_matched = true; let value = value.try_to_bool().unwrap_or_else(|| { span_bug!(test.span, "expected boolean value but got {value:?}") @@ -217,9 +214,9 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { ( &TestKind::Len { len: test_len, op: BinOp::Eq }, - &TestCase::Slice { len, variable_length }, + &TestableCase::Slice { len, variable_length }, ) => { - match (test_len.cmp(&(len as u64)), variable_length) { + match (test_len.cmp(&len), variable_length) { (Ordering::Equal, false) => { // on true, min_len = len = $actual_length, // on false, len != $actual_length @@ -249,10 +246,10 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } ( &TestKind::Len { len: test_len, op: BinOp::Ge }, - &TestCase::Slice { len, variable_length }, + &TestableCase::Slice { len, variable_length }, ) => { // the test is `$actual_len >= test_len` - match (test_len.cmp(&(len as u64)), variable_length) { + match (test_len.cmp(&len), variable_length) { (Ordering::Equal, true) => { // $actual_len >= test_len = pat_len, // so we can match. @@ -281,7 +278,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } } - (TestKind::Range(test), TestCase::Range(pat)) => { + (TestKind::Range(test), TestableCase::Range(pat)) => { if test == pat { fully_matched = true; Some(TestBranch::Success) @@ -292,7 +289,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { if !test.overlaps(pat, self.tcx)? { Some(TestBranch::Failure) } else { None } } } - (TestKind::Range(range), &TestCase::Constant { value }) => { + ( + TestKind::Range(range), + &TestableCase::Constant { + value, + kind: PatConstKind::Bool | PatConstKind::IntOrChar | PatConstKind::Float, + }, + ) => { fully_matched = false; if !range.contains(value, self.tcx)? { // `value` is not contained in the testing range, @@ -303,7 +306,13 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } } - (TestKind::Eq { value: test_val, .. }, TestCase::Constant { value: case_val }) => { + ( + TestKind::Eq { value: test_val, .. }, + TestableCase::Constant { + value: case_val, + kind: PatConstKind::Float | PatConstKind::Other, + }, + ) => { if test_val == case_val { fully_matched = true; Some(TestBranch::Success) @@ -313,7 +322,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } } - (TestKind::Deref { temp: test_temp, .. }, TestCase::Deref { temp, .. }) + (TestKind::Deref { temp: test_temp, .. }, TestableCase::Deref { temp, .. }) if test_temp == temp => { fully_matched = true; diff --git a/compiler/rustc_mir_build/src/builder/matches/match_pair.rs b/compiler/rustc_mir_build/src/builder/matches/match_pair.rs index a0d54354a9c6..798110c8b090 100644 --- a/compiler/rustc_mir_build/src/builder/matches/match_pair.rs +++ b/compiler/rustc_mir_build/src/builder/matches/match_pair.rs @@ -1,14 +1,15 @@ use std::sync::Arc; use rustc_abi::FieldIdx; -use rustc_hir::ByRef; use rustc_middle::mir::*; use rustc_middle::thir::*; -use rustc_middle::ty::{self, Pinnedness, Ty, TypeVisitableExt}; +use rustc_middle::ty::{self, Ty, TypeVisitableExt}; use crate::builder::Builder; use crate::builder::expr::as_place::{PlaceBase, PlaceBuilder}; -use crate::builder::matches::{FlatPat, MatchPairTree, PatternExtraData, TestCase}; +use crate::builder::matches::{ + FlatPat, MatchPairTree, PatConstKind, PatternExtraData, TestableCase, +}; impl<'a, 'tcx> Builder<'a, 'tcx> { /// Builds and pushes [`MatchPairTree`] subtrees, one for each pattern in @@ -132,7 +133,7 @@ impl<'tcx> MatchPairTree<'tcx> { let place = place_builder.try_to_place(cx); let mut subpairs = Vec::new(); - let test_case = match pattern.kind { + let testable_case = match pattern.kind { PatKind::Missing | PatKind::Wild | PatKind::Error(_) => None, PatKind::Or { ref pats } => { @@ -146,18 +147,40 @@ impl<'tcx> MatchPairTree<'tcx> { // FIXME(@dianne): this needs updating/removing if we always merge or-patterns extra_data.bindings.push(super::SubpatternBindings::FromOrPattern); } - Some(TestCase::Or { pats }) + Some(TestableCase::Or { pats }) } PatKind::Range(ref range) => { if range.is_full_range(cx.tcx) == Some(true) { None } else { - Some(TestCase::Range(Arc::clone(range))) + Some(TestableCase::Range(Arc::clone(range))) } } - PatKind::Constant { value } => Some(TestCase::Constant { value }), + PatKind::Constant { value } => { + // CAUTION: The type of the pattern node (`pattern.ty`) is + // _often_ the same as the type of the const value (`value.ty`), + // but there are some cases where those types differ + // (e.g. when `deref!(..)` patterns interact with `String`). + + // Classify the constant-pattern into further kinds, to + // reduce the number of ad-hoc type tests needed later on. + let pat_ty = pattern.ty; + let const_kind = if pat_ty.is_bool() { + PatConstKind::Bool + } else if pat_ty.is_integral() || pat_ty.is_char() { + PatConstKind::IntOrChar + } else if pat_ty.is_floating_point() { + PatConstKind::Float + } else { + // FIXME(Zalathar): This still covers several different + // categories (e.g. raw pointer, string, pattern-type) + // which could be split out into their own kinds. + PatConstKind::Other + }; + Some(TestableCase::Constant { value, kind: const_kind }) + } PatKind::AscribeUserType { ascription: Ascription { ref annotation, variance }, @@ -256,8 +279,8 @@ impl<'tcx> MatchPairTree<'tcx> { if prefix.is_empty() && slice.is_some() && suffix.is_empty() { None } else { - Some(TestCase::Slice { - len: prefix.len() + suffix.len(), + Some(TestableCase::Slice { + len: u64::try_from(prefix.len() + suffix.len()).unwrap(), variable_length: slice.is_some(), }) } @@ -275,7 +298,11 @@ impl<'tcx> MatchPairTree<'tcx> { cx.def_id.into(), ) }) && !adt_def.variant_list_has_applicable_non_exhaustive(); - if irrefutable { None } else { Some(TestCase::Variant { adt_def, variant_index }) } + if irrefutable { + None + } else { + Some(TestableCase::Variant { adt_def, variant_index }) + } } PatKind::Leaf { ref subpatterns } => { @@ -283,8 +310,9 @@ impl<'tcx> MatchPairTree<'tcx> { None } + // FIXME: Pin-patterns should probably have their own pattern kind, + // instead of overloading `PatKind::Deref` via the pattern type. PatKind::Deref { ref subpattern } - | PatKind::DerefPattern { ref subpattern, borrow: ByRef::Yes(Pinnedness::Pinned, _) } if let Some(ref_ty) = pattern.ty.pinned_ty() && ref_ty.is_ref() => { @@ -298,12 +326,8 @@ impl<'tcx> MatchPairTree<'tcx> { None } - PatKind::DerefPattern { borrow: ByRef::Yes(Pinnedness::Pinned, _), .. } => { - rustc_middle::bug!("RefPin pattern on non-`Pin` type {:?}", pattern.ty) - } - PatKind::Deref { ref subpattern } - | PatKind::DerefPattern { ref subpattern, borrow: ByRef::No } => { + | PatKind::DerefPattern { ref subpattern, borrow: DerefPatBorrowMode::Box } => { MatchPairTree::for_pattern( place_builder.deref(), subpattern, @@ -316,7 +340,7 @@ impl<'tcx> MatchPairTree<'tcx> { PatKind::DerefPattern { ref subpattern, - borrow: ByRef::Yes(Pinnedness::Not, mutability), + borrow: DerefPatBorrowMode::Borrow(mutability), } => { // Create a new temporary for each deref pattern. // FIXME(deref_patterns): dedup temporaries to avoid multiple `deref()` calls? @@ -331,17 +355,23 @@ impl<'tcx> MatchPairTree<'tcx> { &mut subpairs, extra_data, ); - Some(TestCase::Deref { temp, mutability }) + Some(TestableCase::Deref { temp, mutability }) } - PatKind::Never => Some(TestCase::Never), + PatKind::Never => Some(TestableCase::Never), }; - if let Some(test_case) = test_case { + if let Some(testable_case) = testable_case { // This pattern is refutable, so push a new match-pair node. + // + // Note: unless test_case is TestCase::Or, place must not be None. + // This means that the closure capture analysis in + // rustc_hir_typeck::upvar, and in particular the pattern handling + // code of ExprUseVisitor, must capture all of the places we'll use. + // Make sure to keep these two parts in sync! match_pairs.push(MatchPairTree { place, - test_case, + testable_case, subpairs, pattern_ty: pattern.ty, pattern_span: pattern.span, diff --git a/compiler/rustc_mir_build/src/builder/matches/mod.rs b/compiler/rustc_mir_build/src/builder/matches/mod.rs index 5074ce336798..1da983462887 100644 --- a/compiler/rustc_mir_build/src/builder/matches/mod.rs +++ b/compiler/rustc_mir_build/src/builder/matches/mod.rs @@ -11,10 +11,10 @@ use std::mem; use std::sync::Arc; use itertools::{Itertools, Position}; -use rustc_abi::{FIRST_VARIANT, FieldIdx, VariantIdx}; +use rustc_abi::{FIRST_VARIANT, VariantIdx}; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::stack::ensure_sufficient_stack; -use rustc_hir::{BindingMode, ByRef, LangItem, LetStmt, LocalSource, Node, Pinnedness}; +use rustc_hir::{BindingMode, ByRef, LangItem, LetStmt, LocalSource, Node}; use rustc_middle::middle::region::{self, TempLifetime}; use rustc_middle::mir::*; use rustc_middle::thir::{self, *}; @@ -920,10 +920,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { visit_subpat(self, subpattern, &user_tys.deref(), f); } - PatKind::DerefPattern { ref subpattern, borrow: ByRef::Yes(Pinnedness::Pinned, _) } => { - visit_subpat(self, subpattern, &user_tys.leaf(FieldIdx::ZERO).deref(), f); - } - PatKind::DerefPattern { ref subpattern, .. } => { visit_subpat(self, subpattern, &ProjectedUserTypesNode::None, f); } @@ -1078,7 +1074,7 @@ struct Candidate<'tcx> { /// (see [`Builder::test_remaining_match_pairs_after_or`]). /// /// Invariants: - /// - All or-patterns ([`TestCase::Or`]) have been sorted to the end. + /// - All or-patterns ([`TestableCase::Or`]) have been sorted to the end. match_pairs: Vec>, /// ...and if this is non-empty, one of these subcandidates also has to match... @@ -1164,12 +1160,15 @@ impl<'tcx> Candidate<'tcx> { /// Restores the invariant that or-patterns must be sorted to the end. fn sort_match_pairs(&mut self) { - self.match_pairs.sort_by_key(|pair| matches!(pair.test_case, TestCase::Or { .. })); + self.match_pairs.sort_by_key(|pair| matches!(pair.testable_case, TestableCase::Or { .. })); } /// Returns whether the first match pair of this candidate is an or-pattern. fn starts_with_or_pattern(&self) -> bool { - matches!(&*self.match_pairs, [MatchPairTree { test_case: TestCase::Or { .. }, .. }, ..]) + matches!( + &*self.match_pairs, + [MatchPairTree { testable_case: TestableCase::Or { .. }, .. }, ..] + ) } /// Visit the leaf candidates (those with no subcandidates) contained in @@ -1261,22 +1260,44 @@ struct Ascription<'tcx> { /// Instead they participate in or-pattern expansion, where they are transformed into /// subcandidates. See [`Builder::expand_and_match_or_candidates`]. #[derive(Debug, Clone)] -enum TestCase<'tcx> { +enum TestableCase<'tcx> { Variant { adt_def: ty::AdtDef<'tcx>, variant_index: VariantIdx }, - Constant { value: ty::Value<'tcx> }, + Constant { value: ty::Value<'tcx>, kind: PatConstKind }, Range(Arc>), - Slice { len: usize, variable_length: bool }, + Slice { len: u64, variable_length: bool }, Deref { temp: Place<'tcx>, mutability: Mutability }, Never, Or { pats: Box<[FlatPat<'tcx>]> }, } -impl<'tcx> TestCase<'tcx> { +impl<'tcx> TestableCase<'tcx> { fn as_range(&self) -> Option<&PatRange<'tcx>> { if let Self::Range(v) = self { Some(v.as_ref()) } else { None } } } +/// Sub-classification of [`TestableCase::Constant`], which helps to avoid +/// some redundant ad-hoc checks when preparing and lowering tests. +#[derive(Debug, Clone)] +enum PatConstKind { + /// The primitive `bool` type, which is like an integer but simpler, + /// having only two values. + Bool, + /// Primitive unsigned/signed integer types, plus `char`. + /// These types interact nicely with `SwitchInt`. + IntOrChar, + /// Floating-point primitives, e.g. `f32`, `f64`. + /// These types don't support `SwitchInt` and require an equality test, + /// but can also interact with range pattern tests. + Float, + /// Any other constant-pattern is usually tested via some kind of equality + /// check. Types that might be encountered here include: + /// - `&str` + /// - raw pointers derived from integer values + /// - pattern types, e.g. `pattern_type!(u32 is 1..)` + Other, +} + /// Node in a tree of "match pairs", where each pair consists of a place to be /// tested, and a test to perform on that place. /// @@ -1289,12 +1310,12 @@ pub(crate) struct MatchPairTree<'tcx> { /// --- /// This can be `None` if it referred to a non-captured place in a closure. /// - /// Invariant: Can only be `None` when `test_case` is `Or`. + /// Invariant: Can only be `None` when `testable_case` is `Or`. /// Therefore this must be `Some(_)` after or-pattern expansion. place: Option>, /// ... must pass this test... - test_case: TestCase<'tcx>, + testable_case: TestableCase<'tcx>, /// ... and these subpairs must match. /// @@ -1317,7 +1338,7 @@ enum TestKind<'tcx> { /// Test what enum variant a value is. /// /// The subset of expected variants is not stored here; instead they are - /// extracted from the [`TestCase`]s of the candidates participating in the + /// extracted from the [`TestableCase`]s of the candidates participating in the /// test. Switch { /// The enum type being tested. @@ -1327,7 +1348,7 @@ enum TestKind<'tcx> { /// Test what value an integer or `char` has. /// /// The test's target values are not stored here; instead they are extracted - /// from the [`TestCase`]s of the candidates participating in the test. + /// from the [`TestableCase`]s of the candidates participating in the test. SwitchInt, /// Test whether a `bool` is `true` or `false`. @@ -1948,7 +1969,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { candidate: &mut Candidate<'tcx>, match_pair: MatchPairTree<'tcx>, ) { - let TestCase::Or { pats } = match_pair.test_case else { bug!() }; + let TestableCase::Or { pats } = match_pair.testable_case else { bug!() }; debug!("expanding or-pattern: candidate={:#?}\npats={:#?}", candidate, pats); candidate.or_span = Some(match_pair.pattern_span); candidate.subcandidates = pats @@ -2116,7 +2137,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { debug_assert!( remaining_match_pairs .iter() - .all(|match_pair| matches!(match_pair.test_case, TestCase::Or { .. })) + .all(|match_pair| matches!(match_pair.testable_case, TestableCase::Or { .. })) ); // Visit each leaf candidate within this subtree, add a copy of the remaining @@ -2936,7 +2957,8 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { bug!("malformed valtree for an enum") }; - let ValTreeKind::Leaf(actual_variant_idx) = ***actual_variant_idx else { + let ValTreeKind::Leaf(actual_variant_idx) = *actual_variant_idx.to_value().valtree + else { bug!("malformed valtree for an enum") }; @@ -2944,7 +2966,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } Constructor::IntRange(int_range) => { let size = pat.ty().primitive_size(self.tcx); - let actual_int = valtree.unwrap_leaf().to_bits(size); + let actual_int = valtree.to_leaf().to_bits(size); let actual_int = if pat.ty().is_signed() { MaybeInfiniteInt::new_finite_int(actual_int, size.bits()) } else { @@ -2952,33 +2974,33 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { }; IntRange::from_singleton(actual_int).is_subrange(int_range) } - Constructor::Bool(pattern_value) => match valtree.unwrap_leaf().try_to_bool() { + Constructor::Bool(pattern_value) => match valtree.to_leaf().try_to_bool() { Ok(actual_value) => *pattern_value == actual_value, Err(()) => bug!("bool value with invalid bits"), }, Constructor::F16Range(l, h, end) => { - let actual = valtree.unwrap_leaf().to_f16(); + let actual = valtree.to_leaf().to_f16(); match end { RangeEnd::Included => (*l..=*h).contains(&actual), RangeEnd::Excluded => (*l..*h).contains(&actual), } } Constructor::F32Range(l, h, end) => { - let actual = valtree.unwrap_leaf().to_f32(); + let actual = valtree.to_leaf().to_f32(); match end { RangeEnd::Included => (*l..=*h).contains(&actual), RangeEnd::Excluded => (*l..*h).contains(&actual), } } Constructor::F64Range(l, h, end) => { - let actual = valtree.unwrap_leaf().to_f64(); + let actual = valtree.to_leaf().to_f64(); match end { RangeEnd::Included => (*l..=*h).contains(&actual), RangeEnd::Excluded => (*l..*h).contains(&actual), } } Constructor::F128Range(l, h, end) => { - let actual = valtree.unwrap_leaf().to_f128(); + let actual = valtree.to_leaf().to_f128(); match end { RangeEnd::Included => (*l..=*h).contains(&actual), RangeEnd::Excluded => (*l..*h).contains(&actual), diff --git a/compiler/rustc_mir_build/src/builder/matches/test.rs b/compiler/rustc_mir_build/src/builder/matches/test.rs index 46a76a7b9a30..4a2823bf5d6b 100644 --- a/compiler/rustc_mir_build/src/builder/matches/test.rs +++ b/compiler/rustc_mir_build/src/builder/matches/test.rs @@ -19,7 +19,9 @@ use rustc_span::{DUMMY_SP, Span, Symbol, sym}; use tracing::{debug, instrument}; use crate::builder::Builder; -use crate::builder::matches::{MatchPairTree, Test, TestBranch, TestCase, TestKind}; +use crate::builder::matches::{ + MatchPairTree, PatConstKind, Test, TestBranch, TestKind, TestableCase, +}; impl<'a, 'tcx> Builder<'a, 'tcx> { /// Identifies what test is needed to decide if `match_pair` is applicable. @@ -29,30 +31,37 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { &mut self, match_pair: &MatchPairTree<'tcx>, ) -> Test<'tcx> { - let kind = match match_pair.test_case { - TestCase::Variant { adt_def, variant_index: _ } => TestKind::Switch { adt_def }, + let kind = match match_pair.testable_case { + TestableCase::Variant { adt_def, variant_index: _ } => TestKind::Switch { adt_def }, - TestCase::Constant { .. } if match_pair.pattern_ty.is_bool() => TestKind::If, - TestCase::Constant { .. } if is_switch_ty(match_pair.pattern_ty) => TestKind::SwitchInt, - TestCase::Constant { value } => TestKind::Eq { value, cast_ty: match_pair.pattern_ty }, + TestableCase::Constant { value: _, kind: PatConstKind::Bool } => TestKind::If, + TestableCase::Constant { value: _, kind: PatConstKind::IntOrChar } => { + TestKind::SwitchInt + } + TestableCase::Constant { value, kind: PatConstKind::Float } => { + TestKind::Eq { value, cast_ty: match_pair.pattern_ty } + } + TestableCase::Constant { value, kind: PatConstKind::Other } => { + TestKind::Eq { value, cast_ty: match_pair.pattern_ty } + } - TestCase::Range(ref range) => { + TestableCase::Range(ref range) => { assert_eq!(range.ty, match_pair.pattern_ty); TestKind::Range(Arc::clone(range)) } - TestCase::Slice { len, variable_length } => { + TestableCase::Slice { len, variable_length } => { let op = if variable_length { BinOp::Ge } else { BinOp::Eq }; - TestKind::Len { len: len as u64, op } + TestKind::Len { len, op } } - TestCase::Deref { temp, mutability } => TestKind::Deref { temp, mutability }, + TestableCase::Deref { temp, mutability } => TestKind::Deref { temp, mutability }, - TestCase::Never => TestKind::Never, + TestableCase::Never => TestKind::Never, // Or-patterns are not tested directly; instead they are expanded into subcandidates, // which are then distinguished by testing whatever non-or patterns they contain. - TestCase::Or { .. } => bug!("or-patterns should have already been handled"), + TestableCase::Or { .. } => bug!("or-patterns should have already been handled"), }; Test { span: match_pair.pattern_span, kind } @@ -112,7 +121,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { let switch_targets = SwitchTargets::new( target_blocks.iter().filter_map(|(&branch, &block)| { if let TestBranch::Constant(value) = branch { - let bits = value.valtree.unwrap_leaf().to_bits_unchecked(); + let bits = value.to_leaf().to_bits_unchecked(); Some((bits, block)) } else { None @@ -487,11 +496,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { } } -/// Returns true if this type be used with [`TestKind::SwitchInt`]. -pub(crate) fn is_switch_ty(ty: Ty<'_>) -> bool { - ty.is_integral() || ty.is_char() -} - fn trait_method<'tcx>( tcx: TyCtxt<'tcx>, trait_def_id: DefId, diff --git a/compiler/rustc_mir_build/src/builder/matches/util.rs b/compiler/rustc_mir_build/src/builder/matches/util.rs index 2c8ad95b6afd..f1df90f93fd6 100644 --- a/compiler/rustc_mir_build/src/builder/matches/util.rs +++ b/compiler/rustc_mir_build/src/builder/matches/util.rs @@ -6,7 +6,7 @@ use tracing::debug; use crate::builder::Builder; use crate::builder::expr::as_place::PlaceBase; -use crate::builder::matches::{Binding, Candidate, FlatPat, MatchPairTree, TestCase}; +use crate::builder::matches::{Binding, Candidate, FlatPat, MatchPairTree, TestableCase}; impl<'a, 'tcx> Builder<'a, 'tcx> { /// Creates a false edge to `imaginary_target` and a real edge to @@ -159,11 +159,11 @@ impl<'a, 'b, 'tcx> FakeBorrowCollector<'a, 'b, 'tcx> { } fn visit_match_pair(&mut self, match_pair: &MatchPairTree<'tcx>) { - if let TestCase::Or { pats, .. } = &match_pair.test_case { + if let TestableCase::Or { pats, .. } = &match_pair.testable_case { for flat_pat in pats.iter() { self.visit_flat_pat(flat_pat) } - } else if matches!(match_pair.test_case, TestCase::Deref { .. }) { + } else if matches!(match_pair.testable_case, TestableCase::Deref { .. }) { // The subpairs of a deref pattern are all places relative to the deref temporary, so we // don't fake borrow them. Problem is, if we only shallowly fake-borrowed // `match_pair.place`, this would allow: diff --git a/compiler/rustc_mir_build/src/builder/mod.rs b/compiler/rustc_mir_build/src/builder/mod.rs index 14a24265a8f4..40f1fe0d5d11 100644 --- a/compiler/rustc_mir_build/src/builder/mod.rs +++ b/compiler/rustc_mir_build/src/builder/mod.rs @@ -838,26 +838,6 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { self.parent_module, self.infcx.typing_env(self.param_env), ); - - // check if the function's return type is inhabited - // this was added here because of this regression - // https://github.com/rust-lang/rust/issues/149571 - let output_is_inhabited = - if matches!(self.tcx.def_kind(self.def_id), DefKind::Fn | DefKind::AssocFn) { - self.tcx - .fn_sig(self.def_id) - .instantiate_identity() - .skip_binder() - .output() - .is_inhabited_from( - self.tcx, - self.parent_module, - self.infcx.typing_env(self.param_env), - ) - } else { - true - }; - if !ty_is_inhabited { // Unreachable code warnings are already emitted during type checking. // However, during type checking, full type information is being @@ -868,7 +848,23 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // uninhabited types (e.g. empty enums). The check above is used so // that we do not emit the same warning twice if the uninhabited type // is indeed `!`. - if !ty.is_never() && output_is_inhabited { + if !ty.is_never() + && matches!(self.tcx.def_kind(self.def_id), DefKind::Fn | DefKind::AssocFn) + // check if the function's return type is inhabited + // this was added here because of this regression + // https://github.com/rust-lang/rust/issues/149571 + && self + .tcx + .fn_sig(self.def_id) + .instantiate_identity() + .skip_binder() + .output() + .is_inhabited_from( + self.tcx, + self.parent_module, + self.infcx.typing_env(self.param_env), + ) + { lints.push((target_bb, ty, term.source_info.span)); } diff --git a/compiler/rustc_mir_build/src/builder/scope.rs b/compiler/rustc_mir_build/src/builder/scope.rs index 966598736220..981704052536 100644 --- a/compiler/rustc_mir_build/src/builder/scope.rs +++ b/compiler/rustc_mir_build/src/builder/scope.rs @@ -897,7 +897,14 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { self.tcx, ValTree::from_branches( self.tcx, - [ValTree::from_scalar_int(self.tcx, variant_index.as_u32().into())], + [ty::Const::new_value( + self.tcx, + ValTree::from_scalar_int( + self.tcx, + variant_index.as_u32().into(), + ), + self.tcx.types.u32, + )], ), self.thir[value].ty, ), @@ -1099,7 +1106,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { Some(DropData { source_info, local, kind: DropKind::Value }) } - Operand::Constant(_) => None, + Operand::Constant(_) | Operand::RuntimeChecks(_) => None, }) .collect(); @@ -1563,7 +1570,7 @@ impl<'a, 'tcx> Builder<'a, 'tcx> { // look for moves of a local variable, like `MOVE(_X)` let locals_moved = operands.iter().flat_map(|operand| match operand.node { - Operand::Copy(_) | Operand::Constant(_) => None, + Operand::Copy(_) | Operand::Constant(_) | Operand::RuntimeChecks(_) => None, Operand::Move(place) => place.as_local(), }); diff --git a/compiler/rustc_mir_build/src/thir/constant.rs b/compiler/rustc_mir_build/src/thir/constant.rs index 6e071fb344c4..563212a51f31 100644 --- a/compiler/rustc_mir_build/src/thir/constant.rs +++ b/compiler/rustc_mir_build/src/thir/constant.rs @@ -63,7 +63,7 @@ pub(crate) fn lit_to_const<'tcx>( // A CStr is a newtype around a byte slice, so we create the inner slice here. // We need a branch for each "level" of the data structure. let bytes = ty::ValTree::from_raw_bytes(tcx, byte_sym.as_byte_str()); - ty::ValTree::from_branches(tcx, [bytes]) + ty::ValTree::from_branches(tcx, [ty::Const::new_value(tcx, bytes, *inner_ty)]) } (ast::LitKind::Int(n, _), ty::Uint(ui)) if !neg => { let scalar_int = trunc(n.get(), *ui); diff --git a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs index 6316ccf1b8c5..ce4c89a8eb2e 100644 --- a/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs +++ b/compiler/rustc_mir_build/src/thir/pattern/const_to_pat.rs @@ -28,7 +28,7 @@ use crate::errors::{ PointerPattern, TypeNotPartialEq, TypeNotStructural, UnionPattern, UnsizedPattern, }; -impl<'a, 'tcx> PatCtxt<'a, 'tcx> { +impl<'tcx> PatCtxt<'tcx> { /// Converts a constant to a pattern (if possible). /// This means aggregate values (like structs and enums) are converted /// to a pattern that matches the value (as if you'd compared via structural equality). @@ -64,7 +64,7 @@ struct ConstToPat<'tcx> { } impl<'tcx> ConstToPat<'tcx> { - fn new(pat_ctxt: &PatCtxt<'_, 'tcx>, id: hir::HirId, span: Span, c: ty::Const<'tcx>) -> Self { + fn new(pat_ctxt: &PatCtxt<'tcx>, id: hir::HirId, span: Span, c: ty::Const<'tcx>) -> Self { trace!(?pat_ctxt.typeck_results.hir_owner); ConstToPat { tcx: pat_ctxt.tcx, typing_env: pat_ctxt.typing_env, span, id, c } } @@ -239,14 +239,14 @@ impl<'tcx> ConstToPat<'tcx> { return self.mk_err(tcx.dcx().create_err(err), ty); } ty::Adt(adt_def, args) if adt_def.is_enum() => { - let (&variant_index, fields) = cv.unwrap_branch().split_first().unwrap(); - let variant_index = VariantIdx::from_u32(variant_index.unwrap_leaf().to_u32()); + let (&variant_index, fields) = cv.to_branch().split_first().unwrap(); + let variant_index = VariantIdx::from_u32(variant_index.to_leaf().to_u32()); PatKind::Variant { adt_def: *adt_def, args, variant_index, subpatterns: self.field_pats( - fields.iter().copied().zip( + fields.iter().map(|ct| ct.to_value().valtree).zip( adt_def.variants()[variant_index] .fields .iter() @@ -258,28 +258,32 @@ impl<'tcx> ConstToPat<'tcx> { ty::Adt(def, args) => { assert!(!def.is_union()); // Valtree construction would never succeed for unions. PatKind::Leaf { - subpatterns: self.field_pats(cv.unwrap_branch().iter().copied().zip( - def.non_enum_variant().fields.iter().map(|field| field.ty(tcx, args)), - )), + subpatterns: self.field_pats( + cv.to_branch().iter().map(|ct| ct.to_value().valtree).zip( + def.non_enum_variant().fields.iter().map(|field| field.ty(tcx, args)), + ), + ), } } ty::Tuple(fields) => PatKind::Leaf { - subpatterns: self.field_pats(cv.unwrap_branch().iter().copied().zip(fields.iter())), + subpatterns: self.field_pats( + cv.to_branch().iter().map(|ct| ct.to_value().valtree).zip(fields.iter()), + ), }, ty::Slice(elem_ty) => PatKind::Slice { prefix: cv - .unwrap_branch() + .to_branch() .iter() - .map(|val| *self.valtree_to_pat(*val, *elem_ty)) + .map(|val| *self.valtree_to_pat(val.to_value().valtree, *elem_ty)) .collect(), slice: None, suffix: Box::new([]), }, ty::Array(elem_ty, _) => PatKind::Array { prefix: cv - .unwrap_branch() + .to_branch() .iter() - .map(|val| *self.valtree_to_pat(*val, *elem_ty)) + .map(|val| *self.valtree_to_pat(val.to_value().valtree, *elem_ty)) .collect(), slice: None, suffix: Box::new([]), @@ -312,7 +316,7 @@ impl<'tcx> ConstToPat<'tcx> { } }, ty::Float(flt) => { - let v = cv.unwrap_leaf(); + let v = cv.to_leaf(); let is_nan = match flt { ty::FloatTy::F16 => v.to_f16().is_nan(), ty::FloatTy::F32 => v.to_f32().is_nan(), diff --git a/compiler/rustc_mir_build/src/thir/pattern/mod.rs b/compiler/rustc_mir_build/src/thir/pattern/mod.rs index 02e6f8d6ce71..d62faae44d67 100644 --- a/compiler/rustc_mir_build/src/thir/pattern/mod.rs +++ b/compiler/rustc_mir_build/src/thir/pattern/mod.rs @@ -11,16 +11,15 @@ use rustc_abi::{FieldIdx, Integer}; use rustc_errors::codes::*; use rustc_hir::def::{CtorOf, DefKind, Res}; use rustc_hir::pat_util::EnumerateAndAdjustIterator; -use rustc_hir::{self as hir, ByRef, LangItem, Mutability, Pinnedness, RangeEnd}; +use rustc_hir::{self as hir, LangItem, RangeEnd}; use rustc_index::Idx; -use rustc_infer::infer::TyCtxtInferExt; use rustc_middle::mir::interpret::LitToConstInput; use rustc_middle::thir::{ - Ascription, FieldPat, LocalVarId, Pat, PatKind, PatRange, PatRangeBoundary, + Ascription, DerefPatBorrowMode, FieldPat, LocalVarId, Pat, PatKind, PatRange, PatRangeBoundary, }; use rustc_middle::ty::adjustment::{PatAdjust, PatAdjustment}; use rustc_middle::ty::layout::IntegerExt; -use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty, TyCtxt, TypingMode}; +use rustc_middle::ty::{self, CanonicalUserTypeAnnotation, Ty, TyCtxt}; use rustc_middle::{bug, span_bug}; use rustc_span::def_id::DefId; use rustc_span::{ErrorGuaranteed, Span}; @@ -30,19 +29,20 @@ pub(crate) use self::check_match::check_match; use self::migration::PatMigration; use crate::errors::*; -struct PatCtxt<'a, 'tcx> { +/// Context for lowering HIR patterns to THIR patterns. +struct PatCtxt<'tcx> { tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>, - typeck_results: &'a ty::TypeckResults<'tcx>, + typeck_results: &'tcx ty::TypeckResults<'tcx>, /// Used by the Rust 2024 migration lint. - rust_2024_migration: Option>, + rust_2024_migration: Option>, } -pub(super) fn pat_from_hir<'a, 'tcx>( +pub(super) fn pat_from_hir<'tcx>( tcx: TyCtxt<'tcx>, typing_env: ty::TypingEnv<'tcx>, - typeck_results: &'a ty::TypeckResults<'tcx>, + typeck_results: &'tcx ty::TypeckResults<'tcx>, pat: &'tcx hir::Pat<'tcx>, ) -> Box> { let mut pcx = PatCtxt { @@ -62,7 +62,7 @@ pub(super) fn pat_from_hir<'a, 'tcx>( result } -impl<'a, 'tcx> PatCtxt<'a, 'tcx> { +impl<'tcx> PatCtxt<'tcx> { fn lower_pattern(&mut self, pat: &'tcx hir::Pat<'tcx>) -> Box> { let adjustments: &[PatAdjustment<'tcx>] = self.typeck_results.pat_adjustments().get(pat.hir_id).map_or(&[], |v| &**v); @@ -114,16 +114,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> { let borrow = self.typeck_results.deref_pat_borrow_mode(adjust.source, pat); PatKind::DerefPattern { subpattern: thir_pat, borrow } } - PatAdjust::PinDeref => { - let mutable = self.typeck_results.pat_has_ref_mut_binding(pat); - PatKind::DerefPattern { - subpattern: thir_pat, - borrow: ByRef::Yes( - Pinnedness::Pinned, - if mutable { Mutability::Mut } else { Mutability::Not }, - ), - } - } + PatAdjust::PinDeref => PatKind::Deref { subpattern: thir_pat }, }; Box::new(Pat { span, ty: adjust.source, kind }) }); @@ -334,7 +325,7 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> { } hir::PatKind::Box(subpattern) => PatKind::DerefPattern { subpattern: self.lower_pattern(subpattern), - borrow: hir::ByRef::No, + borrow: DerefPatBorrowMode::Box, }, hir::PatKind::Slice(prefix, slice, suffix) => { @@ -621,54 +612,8 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> { pattern } - /// Lowers an inline const block (e.g. `const { 1 + 1 }`) to a pattern. - fn lower_inline_const( - &mut self, - block: &'tcx hir::ConstBlock, - id: hir::HirId, - span: Span, - ) -> PatKind<'tcx> { - let tcx = self.tcx; - let def_id = block.def_id; - let ty = tcx.typeck(def_id).node_type(block.hir_id); - - let typeck_root_def_id = tcx.typeck_root_def_id(def_id.to_def_id()); - let parent_args = ty::GenericArgs::identity_for_item(tcx, typeck_root_def_id); - let args = ty::InlineConstArgs::new(tcx, ty::InlineConstArgsParts { parent_args, ty }).args; - - let ct = ty::UnevaluatedConst { def: def_id.to_def_id(), args }; - let c = ty::Const::new_unevaluated(self.tcx, ct); - let pattern = self.const_to_pat(c, ty, id, span); - - // Apply a type ascription for the inline constant. - let annotation = { - let infcx = tcx.infer_ctxt().build(TypingMode::non_body_analysis()); - let args = ty::InlineConstArgs::new( - tcx, - ty::InlineConstArgsParts { parent_args, ty: infcx.next_ty_var(span) }, - ) - .args; - infcx.canonicalize_user_type_annotation(ty::UserType::new(ty::UserTypeKind::TypeOf( - def_id.to_def_id(), - ty::UserArgs { args, user_self_ty: None }, - ))) - }; - let annotation = - CanonicalUserTypeAnnotation { user_ty: Box::new(annotation), span, inferred_ty: ty }; - PatKind::AscribeUserType { - subpattern: pattern, - ascription: Ascription { - annotation, - // Note that we use `Contravariant` here. See the `variance` field documentation - // for details. - variance: ty::Contravariant, - }, - } - } - /// Lowers the kinds of "expression" that can appear in a HIR pattern: /// - Paths (e.g. `FOO`, `foo::BAR`, `Option::None`) - /// - Inline const blocks (e.g. `const { 1 + 1 }`) /// - Literals, possibly negated (e.g. `-128u8`, `"hello"`) fn lower_pat_expr( &mut self, @@ -677,9 +622,6 @@ impl<'a, 'tcx> PatCtxt<'a, 'tcx> { ) -> PatKind<'tcx> { match &expr.kind { hir::PatExprKind::Path(qpath) => self.lower_path(qpath, expr.hir_id, expr.span).kind, - hir::PatExprKind::ConstBlock(anon_const) => { - self.lower_inline_const(anon_const, expr.hir_id, expr.span) - } hir::PatExprKind::Lit { lit, negated } => { // We handle byte string literal patterns by using the pattern's type instead of the // literal's type in `const_to_pat`: if the literal `b"..."` matches on a slice reference, diff --git a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs index 331e41bd126b..4b2c52ad7999 100644 --- a/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs +++ b/compiler/rustc_mir_dataflow/src/impls/borrowed_locals.rs @@ -91,7 +91,6 @@ where | Rvalue::ThreadLocalRef(..) | Rvalue::Repeat(..) | Rvalue::BinaryOp(..) - | Rvalue::NullaryOp(..) | Rvalue::UnaryOp(..) | Rvalue::Discriminant(..) | Rvalue::Aggregate(..) diff --git a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs index 0cd3405d1e61..ced9bd735ba2 100644 --- a/compiler/rustc_mir_dataflow/src/move_paths/builder.rs +++ b/compiler/rustc_mir_dataflow/src/move_paths/builder.rs @@ -448,10 +448,7 @@ impl<'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> MoveDataBuilder<'a, 'tcx, F> { } } Rvalue::CopyForDeref(..) => unreachable!(), - Rvalue::Ref(..) - | Rvalue::RawPtr(..) - | Rvalue::Discriminant(..) - | Rvalue::NullaryOp(NullOp::RuntimeChecks(_)) => {} + Rvalue::Ref(..) | Rvalue::RawPtr(..) | Rvalue::Discriminant(..) => {} } } @@ -549,9 +546,10 @@ impl<'a, 'tcx, F: Fn(Ty<'tcx>) -> bool> MoveDataBuilder<'a, 'tcx, F> { fn gather_operand(&mut self, operand: &Operand<'tcx>) { match *operand { - Operand::Constant(..) | Operand::Copy(..) => {} // not-a-move + // not-a-move + Operand::Constant(..) | Operand::Copy(..) | Operand::RuntimeChecks(_) => {} + // a move Operand::Move(place) => { - // a move self.gather_move(place); } } diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index daf304c1bf9d..ac94ee5c8104 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -1,3 +1,4 @@ +use std::assert_matches::debug_assert_matches; use std::fmt::{Debug, Formatter}; use std::ops::Range; @@ -350,41 +351,56 @@ pub struct Map<'tcx> { projections: FxHashMap<(PlaceIndex, TrackElem), PlaceIndex>, places: IndexVec>, value_count: usize, + mode: PlaceCollectionMode, // The Range corresponds to a slice into `inner_values_buffer`. inner_values: IndexVec>, inner_values_buffer: Vec, } +#[derive(Copy, Clone, Debug)] +pub enum PlaceCollectionMode { + Full { value_limit: Option }, + OnDemand, +} + impl<'tcx> Map<'tcx> { /// Returns a map that only tracks places whose type has scalar layout. /// /// This is currently the only way to create a [`Map`]. The way in which the tracked places are /// chosen is an implementation detail and may not be relied upon (other than that their type /// are scalars). - pub fn new(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, value_limit: Option) -> Self { + #[tracing::instrument(level = "trace", skip(tcx, body))] + pub fn new(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, mode: PlaceCollectionMode) -> Self { + tracing::trace!(def_id=?body.source.def_id()); + let capacity = 4 * body.local_decls.len(); let mut map = Self { locals: IndexVec::from_elem(None, &body.local_decls), projections: FxHashMap::default(), - places: IndexVec::new(), + places: IndexVec::with_capacity(capacity), value_count: 0, + mode, inner_values: IndexVec::new(), inner_values_buffer: Vec::new(), }; - let exclude = excluded_locals(body); - map.register(tcx, body, exclude, value_limit); + map.register_locals(tcx, body); + match mode { + PlaceCollectionMode::Full { value_limit } => { + map.collect_places(tcx, body); + map.propagate_assignments(tcx, body); + map.create_values(tcx, body, value_limit); + map.trim_useless_places(); + } + PlaceCollectionMode::OnDemand => {} + } debug!("registered {} places ({} nodes in total)", map.value_count, map.places.len()); map } /// Register all non-excluded places that have scalar layout. #[tracing::instrument(level = "trace", skip(self, tcx, body))] - fn register( - &mut self, - tcx: TyCtxt<'tcx>, - body: &Body<'tcx>, - exclude: DenseBitSet, - value_limit: Option, - ) { + fn register_locals(&mut self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { + let exclude = excluded_locals(body); + // Start by constructing the places for each bare local. for (local, decl) in body.local_decls.iter_enumerated() { if exclude.contains(local) { @@ -399,23 +415,85 @@ impl<'tcx> Map<'tcx> { let place = self.places.push(PlaceInfo::new(decl.ty, None)); self.locals[local] = Some(place); } + } - // Collect syntactic places and assignments between them. - let mut collector = - PlaceCollector { tcx, body, map: self, assignments: Default::default() }; + /// Collect syntactic places from body, and create `PlaceIndex` for them. + #[tracing::instrument(level = "trace", skip(self, tcx, body))] + fn collect_places(&mut self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { + let mut collector = PlaceCollector { tcx, body, map: self }; collector.visit_body(body); - let PlaceCollector { mut assignments, .. } = collector; + } - // Just collecting syntactic places is not enough. We may need to propagate this pattern: - // _1 = (const 5u32, const 13i64); - // _2 = _1; - // _3 = (_2.0 as u32); - // - // `_1.0` does not appear, but we still need to track it. This is achieved by propagating - // projections from assignments. We recorded an assignment between `_2` and `_1`, so we - // want `_1` and `_2` to have the same sub-places. - // - // This is what this fixpoint loop does. While we are still creating places, run through + /// Just collecting syntactic places is not enough. We may need to propagate this pattern: + /// _1 = (const 5u32, const 13i64); + /// _2 = _1; + /// _3 = (_2.0 as u32); + /// + /// `_1.0` does not appear, but we still need to track it. This is achieved by propagating + /// projections from assignments. We recorded an assignment between `_2` and `_1`, so we + /// want `_1` and `_2` to have the same sub-places. + #[tracing::instrument(level = "trace", skip(self, tcx, body))] + fn propagate_assignments(&mut self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>) { + // Collect syntactic places and assignments between them. + let mut assignments = FxIndexSet::default(); + + for bbdata in body.basic_blocks.iter() { + for stmt in bbdata.statements.iter() { + let Some((lhs, rhs)) = stmt.kind.as_assign() else { continue }; + match rhs { + Rvalue::Use(Operand::Move(rhs) | Operand::Copy(rhs)) + | Rvalue::CopyForDeref(rhs) => { + let Some(lhs) = self.register_place_and_discr(tcx, body, *lhs) else { + continue; + }; + let Some(rhs) = self.register_place_and_discr(tcx, body, *rhs) else { + continue; + }; + assignments.insert((lhs, rhs)); + } + Rvalue::Aggregate(kind, fields) => { + let Some(mut lhs) = self.register_place_and_discr(tcx, body, *lhs) else { + continue; + }; + match **kind { + // Do not propagate unions. + AggregateKind::Adt(_, _, _, _, Some(_)) => continue, + AggregateKind::Adt(_, variant, _, _, None) => { + let ty = self.places[lhs].ty; + if ty.is_enum() { + lhs = self.register_place_index( + ty, + lhs, + TrackElem::Variant(variant), + ); + } + } + AggregateKind::RawPtr(..) + | AggregateKind::Array(_) + | AggregateKind::Tuple + | AggregateKind::Closure(..) + | AggregateKind::Coroutine(..) + | AggregateKind::CoroutineClosure(..) => {} + } + for (index, field) in fields.iter_enumerated() { + if let Some(rhs) = field.place() + && let Some(rhs) = self.register_place_and_discr(tcx, body, rhs) + { + let lhs = self.register_place_index( + self.places[rhs].ty, + lhs, + TrackElem::Field(index), + ); + assignments.insert((lhs, rhs)); + } + } + } + _ => {} + } + } + } + + // This is a fixpoint loop does. While we are still creating places, run through // all the assignments, and register places for children. let mut num_places = 0; while num_places < self.places.len() { @@ -428,8 +506,11 @@ impl<'tcx> Map<'tcx> { let mut child = self.places[lhs].first_child; while let Some(lhs_child) = child { let PlaceInfo { ty, proj_elem, next_sibling, .. } = self.places[lhs_child]; - let rhs_child = - self.register_place(ty, rhs, proj_elem.expect("child is not a projection")); + let rhs_child = self.register_place_index( + ty, + rhs, + proj_elem.expect("child is not a projection"), + ); assignments.insert((lhs_child, rhs_child)); child = next_sibling; } @@ -438,16 +519,22 @@ impl<'tcx> Map<'tcx> { let mut child = self.places[rhs].first_child; while let Some(rhs_child) = child { let PlaceInfo { ty, proj_elem, next_sibling, .. } = self.places[rhs_child]; - let lhs_child = - self.register_place(ty, lhs, proj_elem.expect("child is not a projection")); + let lhs_child = self.register_place_index( + ty, + lhs, + proj_elem.expect("child is not a projection"), + ); assignments.insert((lhs_child, rhs_child)); child = next_sibling; } } } - drop(assignments); + } - // Create values for places whose type have scalar layout. + /// Create values for places whose type have scalar layout. + #[tracing::instrument(level = "trace", skip(self, tcx, body))] + fn create_values(&mut self, tcx: TyCtxt<'tcx>, body: &Body<'tcx>, value_limit: Option) { + debug_assert_matches!(self.mode, PlaceCollectionMode::Full { .. }); let typing_env = body.typing_env(tcx); for place_info in self.places.iter_mut() { // The user requires a bound on the number of created values. @@ -481,8 +568,12 @@ impl<'tcx> Map<'tcx> { self.cache_preorder_invoke(place); } } + } - // Trim useless places. + /// Trim useless places. + #[tracing::instrument(level = "trace", skip(self))] + fn trim_useless_places(&mut self) { + debug_assert_matches!(self.mode, PlaceCollectionMode::Full { .. }); for opt_place in self.locals.iter_mut() { if let Some(place) = *opt_place && self.inner_values[place].is_empty() @@ -495,7 +586,12 @@ impl<'tcx> Map<'tcx> { } #[tracing::instrument(level = "trace", skip(self), ret)] - fn register_place(&mut self, ty: Ty<'tcx>, base: PlaceIndex, elem: TrackElem) -> PlaceIndex { + pub fn register_place_index( + &mut self, + ty: Ty<'tcx>, + base: PlaceIndex, + elem: TrackElem, + ) -> PlaceIndex { *self.projections.entry((base, elem)).or_insert_with(|| { let next = self.places.push(PlaceInfo::new(ty, Some(elem))); self.places[next].next_sibling = self.places[base].first_child; @@ -504,9 +600,124 @@ impl<'tcx> Map<'tcx> { }) } + #[tracing::instrument(level = "trace", skip(self, tcx, body), ret)] + pub fn register_place( + &mut self, + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + place: Place<'tcx>, + tail: Option, + ) -> Option { + // Create a place for this projection. + let mut place_index = self.locals[place.local]?; + let mut ty = PlaceTy::from_ty(body.local_decls[place.local].ty); + tracing::trace!(?place_index, ?ty); + + for proj in place.projection { + let track_elem = proj.try_into().ok()?; + ty = ty.projection_ty(tcx, proj); + place_index = self.register_place_index(ty.ty, place_index, track_elem); + tracing::trace!(?proj, ?place_index, ?ty); + } + + if let Some(tail) = tail { + let ty = match tail { + TrackElem::Discriminant => ty.ty.discriminant_ty(tcx), + TrackElem::Variant(..) | TrackElem::Field(..) => todo!(), + TrackElem::DerefLen => tcx.types.usize, + }; + place_index = self.register_place_index(ty, place_index, tail); + } + + Some(place_index) + } + + #[tracing::instrument(level = "trace", skip(self, tcx, body), ret)] + fn register_place_and_discr( + &mut self, + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + place: Place<'tcx>, + ) -> Option { + let place = self.register_place(tcx, body, place, None)?; + let ty = self.places[place].ty; + + if let ty::Ref(_, ref_ty, _) | ty::RawPtr(ref_ty, _) = ty.kind() + && let ty::Slice(..) = ref_ty.kind() + { + self.register_place_index(tcx.types.usize, place, TrackElem::DerefLen); + } else if ty.is_enum() { + let discriminant_ty = ty.discriminant_ty(tcx); + self.register_place_index(discriminant_ty, place, TrackElem::Discriminant); + } + + Some(place) + } + + #[tracing::instrument(level = "trace", skip(self, tcx, typing_env), ret)] + pub fn register_value( + &mut self, + tcx: TyCtxt<'tcx>, + typing_env: ty::TypingEnv<'tcx>, + place: PlaceIndex, + ) -> Option { + let place_info = &mut self.places[place]; + if let Some(value) = place_info.value_index { + return Some(value); + } + + if let Ok(ty) = tcx.try_normalize_erasing_regions(typing_env, place_info.ty) { + place_info.ty = ty; + } + + // Allocate a value slot if it doesn't have one, and the user requested one. + if let Ok(layout) = tcx.layout_of(typing_env.as_query_input(place_info.ty)) + && layout.backend_repr.is_scalar() + { + place_info.value_index = Some(self.value_count.into()); + self.value_count += 1; + } + + place_info.value_index + } + + #[tracing::instrument(level = "trace", skip(self, f))] + pub fn register_copy_tree( + &mut self, + // Tree to copy. + source: PlaceIndex, + // Tree to build. + target: PlaceIndex, + f: &mut impl FnMut(ValueIndex, ValueIndex), + ) { + if let Some(source_value) = self.places[source].value_index { + let target_value = *self.places[target].value_index.get_or_insert_with(|| { + let value_index = self.value_count.into(); + self.value_count += 1; + value_index + }); + f(source_value, target_value) + } + + // Iterate over `source` children and recurse. + let mut source_child_iter = self.places[source].first_child; + while let Some(source_child) = source_child_iter { + source_child_iter = self.places[source_child].next_sibling; + + // Try to find corresponding child and recurse. Reasoning is similar as above. + let source_info = &self.places[source_child]; + let source_ty = source_info.ty; + let source_elem = source_info.proj_elem.unwrap(); + let target_child = self.register_place_index(source_ty, target, source_elem); + self.register_copy_tree(source_child, target_child, f); + } + } + /// Precompute the list of values inside `root` and store it inside /// as a slice within `inner_values_buffer`. + #[tracing::instrument(level = "trace", skip(self))] fn cache_preorder_invoke(&mut self, root: PlaceIndex) { + debug_assert_matches!(self.mode, PlaceCollectionMode::Full { .. }); let start = self.inner_values_buffer.len(); if let Some(vi) = self.places[root].value_index { self.inner_values_buffer.push(vi); @@ -528,44 +739,6 @@ struct PlaceCollector<'a, 'tcx> { tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, map: &'a mut Map<'tcx>, - assignments: FxIndexSet<(PlaceIndex, PlaceIndex)>, -} - -impl<'tcx> PlaceCollector<'_, 'tcx> { - #[tracing::instrument(level = "trace", skip(self))] - fn register_place(&mut self, place: Place<'tcx>) -> Option { - // Create a place for this projection. - let mut place_index = self.map.locals[place.local]?; - let mut ty = PlaceTy::from_ty(self.body.local_decls[place.local].ty); - tracing::trace!(?place_index, ?ty); - - if let ty::Ref(_, ref_ty, _) | ty::RawPtr(ref_ty, _) = ty.ty.kind() - && let ty::Slice(..) = ref_ty.kind() - { - self.map.register_place(self.tcx.types.usize, place_index, TrackElem::DerefLen); - } else if ty.ty.is_enum() { - let discriminant_ty = ty.ty.discriminant_ty(self.tcx); - self.map.register_place(discriminant_ty, place_index, TrackElem::Discriminant); - } - - for proj in place.projection { - let track_elem = proj.try_into().ok()?; - ty = ty.projection_ty(self.tcx, proj); - place_index = self.map.register_place(ty.ty, place_index, track_elem); - tracing::trace!(?proj, ?place_index, ?ty); - - if let ty::Ref(_, ref_ty, _) | ty::RawPtr(ref_ty, _) = ty.ty.kind() - && let ty::Slice(..) = ref_ty.kind() - { - self.map.register_place(self.tcx.types.usize, place_index, TrackElem::DerefLen); - } else if ty.ty.is_enum() { - let discriminant_ty = ty.ty.discriminant_ty(self.tcx); - self.map.register_place(discriminant_ty, place_index, TrackElem::Discriminant); - } - } - - Some(place_index) - } } impl<'tcx> Visitor<'tcx> for PlaceCollector<'_, 'tcx> { @@ -575,51 +748,7 @@ impl<'tcx> Visitor<'tcx> for PlaceCollector<'_, 'tcx> { return; } - self.register_place(*place); - } - - fn visit_assign(&mut self, lhs: &Place<'tcx>, rhs: &Rvalue<'tcx>, location: Location) { - self.super_assign(lhs, rhs, location); - - match rhs { - Rvalue::Use(Operand::Move(rhs) | Operand::Copy(rhs)) | Rvalue::CopyForDeref(rhs) => { - let Some(lhs) = self.register_place(*lhs) else { return }; - let Some(rhs) = self.register_place(*rhs) else { return }; - self.assignments.insert((lhs, rhs)); - } - Rvalue::Aggregate(kind, fields) => { - let Some(mut lhs) = self.register_place(*lhs) else { return }; - match **kind { - // Do not propagate unions. - AggregateKind::Adt(_, _, _, _, Some(_)) => return, - AggregateKind::Adt(_, variant, _, _, None) => { - let ty = self.map.places[lhs].ty; - if ty.is_enum() { - lhs = self.map.register_place(ty, lhs, TrackElem::Variant(variant)); - } - } - AggregateKind::RawPtr(..) - | AggregateKind::Array(_) - | AggregateKind::Tuple - | AggregateKind::Closure(..) - | AggregateKind::Coroutine(..) - | AggregateKind::CoroutineClosure(..) => {} - } - for (index, field) in fields.iter_enumerated() { - if let Some(rhs) = field.place() - && let Some(rhs) = self.register_place(rhs) - { - let lhs = self.map.register_place( - self.map.places[rhs].ty, - lhs, - TrackElem::Field(index), - ); - self.assignments.insert((lhs, rhs)); - } - } - } - _ => {} - } + self.map.register_place_and_discr(self.tcx, self.body, *place); } } @@ -694,6 +823,7 @@ impl<'tcx> Map<'tcx> { /// /// `tail_elem` allows to support discriminants that are not a place in MIR, but that we track /// as such. + #[tracing::instrument(level = "trace", skip(self, f))] pub fn for_each_aliasing_place( &self, place: PlaceRef<'_>, @@ -728,6 +858,7 @@ impl<'tcx> Map<'tcx> { } /// Invoke the given function on all the descendants of the given place, except one branch. + #[tracing::instrument(level = "trace", skip(self, f))] fn for_each_variant_sibling( &self, parent: PlaceIndex, @@ -754,9 +885,22 @@ impl<'tcx> Map<'tcx> { } /// Invoke a function on each value in the given place and all descendants. + #[tracing::instrument(level = "trace", skip(self, f))] fn for_each_value_inside(&self, root: PlaceIndex, f: &mut impl FnMut(ValueIndex)) { - for &v in self.values_inside(root) { - f(v) + if let Some(range) = self.inner_values.get(root) { + // Optimized path: we have cached the inner values. + let values = &self.inner_values_buffer[range.clone()]; + for &v in values { + f(v) + } + } else { + if let Some(root) = self.places[root].value_index { + f(root) + } + + for child in self.children(root) { + self.for_each_value_inside(child, f); + } } } @@ -769,7 +913,9 @@ impl<'tcx> Map<'tcx> { f: &mut impl FnMut(PlaceIndex, &O), ) { // Fast path is there is nothing to do. - if self.inner_values[root].is_empty() { + if let Some(value_range) = self.inner_values.get(root) + && value_range.is_empty() + { return; } diff --git a/compiler/rustc_mir_transform/src/cost_checker.rs b/compiler/rustc_mir_transform/src/cost_checker.rs index 1a9af0e22bbe..331c98fc198e 100644 --- a/compiler/rustc_mir_transform/src/cost_checker.rs +++ b/compiler/rustc_mir_transform/src/cost_checker.rs @@ -60,7 +60,27 @@ impl<'b, 'tcx> CostChecker<'b, 'tcx> { } impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { - fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) { + fn visit_operand(&mut self, operand: &Operand<'tcx>, _: Location) { + match operand { + Operand::RuntimeChecks(RuntimeChecks::UbChecks) => { + if !self + .tcx + .sess + .opts + .unstable_opts + .inline_mir_preserve_debug + .unwrap_or(self.tcx.sess.ub_checks()) + { + // If this is in optimized MIR it's because it's used later, so if we don't need UB + // checks this session, give a bonus here to offset the cost of the call later. + self.bonus += CALL_PENALTY; + } + } + _ => {} + } + } + + fn visit_statement(&mut self, statement: &Statement<'tcx>, loc: Location) { // Most costs are in rvalues and terminators, not in statements. match statement.kind { StatementKind::Intrinsic(ref ndi) => { @@ -69,35 +89,13 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { NonDivergingIntrinsic::CopyNonOverlapping(..) => CALL_PENALTY, }; } - _ => self.super_statement(statement, location), + StatementKind::Assign(..) => self.penalty += INSTR_COST, + _ => {} } + self.super_statement(statement, loc) } - fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, _location: Location) { - match rvalue { - // FIXME: Should we do the same for `OverflowChecks`? - Rvalue::NullaryOp(NullOp::RuntimeChecks(RuntimeChecks::UbChecks), ..) - if !self - .tcx - .sess - .opts - .unstable_opts - .inline_mir_preserve_debug - .unwrap_or(self.tcx.sess.ub_checks()) => - { - // If this is in optimized MIR it's because it's used later, - // so if we don't need UB checks this session, give a bonus - // here to offset the cost of the call later. - self.bonus += CALL_PENALTY; - } - // These are essentially constants that didn't end up in an Operand, - // so treat them as also being free. - Rvalue::NullaryOp(..) => {} - _ => self.penalty += INSTR_COST, - } - } - - fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) { + fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, loc: Location) { match &terminator.kind { TerminatorKind::Drop { place, unwind, .. } => { // If the place doesn't actually need dropping, treat it like a regular goto. @@ -126,7 +124,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { self.penalty += CALL_PENALTY; } TerminatorKind::SwitchInt { discr, targets } => { - if discr.constant().is_some() { + if matches!(discr, Operand::Constant(_) | Operand::RuntimeChecks(_)) { // Not only will this become a `Goto`, but likely other // things will be removable as unreachable. self.bonus += CONST_SWITCH_BONUS; @@ -174,6 +172,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { bug!("{kind:?} should not be in runtime MIR"); } } + self.super_terminator(terminator, loc) } } diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs index 69248cf91f24..71bdafa958ca 100644 --- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs +++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs @@ -1,6 +1,7 @@ use rustc_hir::attrs::InlineAttr; use rustc_hir::def::DefKind; use rustc_hir::def_id::LocalDefId; +use rustc_middle::bug; use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; use rustc_middle::query::Providers; @@ -110,6 +111,15 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { && checker.statements <= threshold } +// The threshold that CostChecker computes is balancing the desire to make more things +// inlinable cross crates against the growth in incremental CGU size that happens when too many +// things in the sysroot are made inlinable. +// Permitting calls causes the size of some incremental CGUs to grow, because more functions are +// made inlinable out of the sysroot or dependencies. +// Assert terminators are similar to calls, but do not have the same impact on compile time, so +// those are just treated as statements. +// A threshold exists at all because we don't want to blindly mark a huge function as inlinable. + struct CostChecker<'b, 'tcx> { tcx: TyCtxt<'tcx>, callee_body: &'b Body<'tcx>, @@ -129,9 +139,10 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { } fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) { + self.statements += 1; let tcx = self.tcx; - match terminator.kind { - TerminatorKind::Drop { ref place, unwind, .. } => { + match &terminator.kind { + TerminatorKind::Drop { place, unwind, .. } => { let ty = place.ty(self.callee_body, tcx).ty; if !ty.is_trivially_pure_clone_copy() { self.calls += 1; @@ -140,7 +151,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { } } } - TerminatorKind::Call { ref func, unwind, .. } => { + TerminatorKind::Call { func, unwind, .. } => { // We track calls because they make our function not a leaf (and in theory, the // number of calls indicates how likely this function is to perturb other CGUs). // But intrinsics don't have a body that gets assigned to a CGU, so they are @@ -155,21 +166,31 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { self.landing_pads += 1; } } - TerminatorKind::Assert { unwind, .. } => { + TerminatorKind::TailCall { .. } => { self.calls += 1; + } + TerminatorKind::Assert { unwind, .. } => { if let UnwindAction::Cleanup(_) = unwind { self.landing_pads += 1; } } TerminatorKind::UnwindResume => self.resumes += 1, TerminatorKind::InlineAsm { unwind, .. } => { - self.statements += 1; if let UnwindAction::Cleanup(_) = unwind { self.landing_pads += 1; } } - TerminatorKind::Return => {} - _ => self.statements += 1, + TerminatorKind::Return + | TerminatorKind::Goto { .. } + | TerminatorKind::SwitchInt { .. } + | TerminatorKind::Unreachable + | TerminatorKind::UnwindTerminate(_) => {} + kind @ (TerminatorKind::FalseUnwind { .. } + | TerminatorKind::FalseEdge { .. } + | TerminatorKind::Yield { .. } + | TerminatorKind::CoroutineDrop) => { + bug!("{kind:?} should not be in runtime MIR"); + } } } } diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs index bccdd526ab7f..5254f60a1503 100644 --- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs +++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs @@ -21,7 +21,7 @@ use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_mir_dataflow::fmt::DebugWithContext; use rustc_mir_dataflow::lattice::{FlatSet, HasBottom}; use rustc_mir_dataflow::value_analysis::{ - Map, PlaceIndex, State, TrackElem, ValueOrPlace, debug_with_context, + Map, PlaceCollectionMode, PlaceIndex, State, TrackElem, ValueOrPlace, debug_with_context, }; use rustc_mir_dataflow::{Analysis, ResultsVisitor, visit_reachable_results}; use rustc_span::DUMMY_SP; @@ -55,10 +55,10 @@ impl<'tcx> crate::MirPass<'tcx> for DataflowConstProp { // `O(num_nodes * tracked_places * n)` in terms of time complexity. Since the number of // map nodes is strongly correlated to the number of tracked places, this becomes more or // less `O(n)` if we place a constant limit on the number of tracked places. - let place_limit = if tcx.sess.mir_opt_level() < 4 { Some(PLACE_LIMIT) } else { None }; + let value_limit = if tcx.sess.mir_opt_level() < 4 { Some(PLACE_LIMIT) } else { None }; // Decide which places to track during the analysis. - let map = Map::new(tcx, body, place_limit); + let map = Map::new(tcx, body, PlaceCollectionMode::Full { value_limit }); // Perform the actual dataflow analysis. let const_ = debug_span!("analyze") @@ -211,6 +211,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { state: &mut State>, ) -> ValueOrPlace> { match operand { + Operand::RuntimeChecks(_) => ValueOrPlace::TOP, Operand::Constant(box constant) => { ValueOrPlace::Value(self.handle_constant(constant, state)) } @@ -463,9 +464,6 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { FlatSet::Top => FlatSet::Top, } } - Rvalue::NullaryOp(NullOp::RuntimeChecks(_)) => { - return ValueOrPlace::TOP; - } Rvalue::Discriminant(place) => state.get_discr(place.as_ref(), &self.map), Rvalue::Use(operand) => return self.handle_operand(operand, state), Rvalue::CopyForDeref(_) => bug!("`CopyForDeref` in runtime MIR"), @@ -533,6 +531,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { operand: &Operand<'tcx>, ) { match operand { + Operand::RuntimeChecks(_) => {} Operand::Copy(rhs) | Operand::Move(rhs) => { if let Some(rhs) = self.map.find(rhs.as_ref()) { state.insert_place_idx(place, rhs, &self.map); @@ -1039,7 +1038,7 @@ impl<'tcx> MutVisitor<'tcx> for Patch<'tcx> { self.super_operand(operand, location) } } - Operand::Constant(_) => {} + Operand::Constant(_) | Operand::RuntimeChecks(_) => {} } } diff --git a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs index da88e5c698bf..47d735eed9b1 100644 --- a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs +++ b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs @@ -117,11 +117,7 @@ impl<'tcx> crate::MirPass<'tcx> for EarlyOtherwiseBranch { unreachable!() }; // Always correct since we can only switch on `Copy` types - let parent_op = match parent_op { - Operand::Move(x) => Operand::Copy(*x), - Operand::Copy(x) => Operand::Copy(*x), - Operand::Constant(x) => Operand::Constant(x.clone()), - }; + let parent_op = parent_op.to_copy(); let parent_ty = parent_op.ty(body.local_decls(), tcx); let statements_before = bbs[parent].statements.len(); let parent_end = Location { block: parent, statement_index: statements_before }; diff --git a/compiler/rustc_mir_transform/src/function_item_references.rs b/compiler/rustc_mir_transform/src/function_item_references.rs index 38b5ccdb32e7..2ae8f43cf6c8 100644 --- a/compiler/rustc_mir_transform/src/function_item_references.rs +++ b/compiler/rustc_mir_transform/src/function_item_references.rs @@ -136,12 +136,7 @@ impl<'tcx> FunctionItemRefChecker<'_, 'tcx> { } fn nth_arg_span(&self, args: &[Spanned>], n: usize) -> Span { - match &args[n].node { - Operand::Copy(place) | Operand::Move(place) => { - self.body.local_decls[place.local].source_info.span - } - Operand::Constant(constant) => constant.span, - } + args[n].node.span(&self.body.local_decls) } fn emit_lint( diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index 526d0e96a36d..820998eed100 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -248,7 +248,7 @@ enum Value<'a, 'tcx> { Discriminant(VnIndex), // Operations. - NullaryOp(NullOp), + RuntimeChecks(RuntimeChecks), UnaryOp(UnOp, VnIndex), BinaryOp(BinOp, VnIndex, VnIndex), Cast { @@ -420,6 +420,19 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { self.ecx.typing_env() } + fn insert_unique( + &mut self, + ty: Ty<'tcx>, + value: impl FnOnce(VnOpaque) -> Value<'a, 'tcx>, + ) -> VnIndex { + let index = self.values.insert_unique(ty, value); + let _index = self.evaluated.push(None); + debug_assert_eq!(index, _index); + let _index = self.rev_locals.push(SmallVec::new()); + debug_assert_eq!(index, _index); + index + } + #[instrument(level = "trace", skip(self), ret)] fn insert(&mut self, ty: Ty<'tcx>, value: Value<'a, 'tcx>) -> VnIndex { let (index, new) = self.values.insert(ty, value); @@ -437,11 +450,8 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { /// from all the others. #[instrument(level = "trace", skip(self), ret)] fn new_opaque(&mut self, ty: Ty<'tcx>) -> VnIndex { - let index = self.values.insert_unique(ty, Value::Opaque); - let _index = self.evaluated.push(Some(None)); - debug_assert_eq!(index, _index); - let _index = self.rev_locals.push(SmallVec::new()); - debug_assert_eq!(index, _index); + let index = self.insert_unique(ty, Value::Opaque); + self.evaluated[index] = Some(None); index } @@ -470,42 +480,29 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { projection.map(|proj| proj.try_map(|index| self.locals[index], |ty| ty).ok_or(())); let projection = self.arena.try_alloc_from_iter(projection).ok()?; - let index = self.values.insert_unique(ty, |provenance| Value::Address { + let index = self.insert_unique(ty, |provenance| Value::Address { base, projection, kind, provenance, }); - let _index = self.evaluated.push(None); - debug_assert_eq!(index, _index); - let _index = self.rev_locals.push(SmallVec::new()); - debug_assert_eq!(index, _index); - Some(index) } #[instrument(level = "trace", skip(self), ret)] fn insert_constant(&mut self, value: Const<'tcx>) -> VnIndex { - let (index, new) = if value.is_deterministic() { + if value.is_deterministic() { // The constant is deterministic, no need to disambiguate. let constant = Value::Constant { value, disambiguator: None }; - self.values.insert(value.ty(), constant) + self.insert(value.ty(), constant) } else { // Multiple mentions of this constant will yield different values, // so assign a different `disambiguator` to ensure they do not get the same `VnIndex`. - let index = self.values.insert_unique(value.ty(), |disambiguator| Value::Constant { + self.insert_unique(value.ty(), |disambiguator| Value::Constant { value, disambiguator: Some(disambiguator), - }); - (index, true) - }; - if new { - let _index = self.evaluated.push(None); - debug_assert_eq!(index, _index); - let _index = self.rev_locals.push(SmallVec::new()); - debug_assert_eq!(index, _index); + }) } - index } #[inline] @@ -570,6 +567,8 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { _ if ty.is_zst() => ImmTy::uninit(ty).into(), Opaque(_) => return None, + // Keep runtime check constants as symbolic. + RuntimeChecks(..) => return None, // In general, evaluating repeat expressions just consumes a lot of memory. // But in the special case that the element is just Immediate::Uninit, we can evaluate @@ -681,7 +680,6 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { self.ecx.discriminant_for_variant(base.layout.ty, variant).discard_err()?; discr_value.into() } - NullaryOp(NullOp::RuntimeChecks(_)) => return None, UnaryOp(un_op, operand) => { let operand = self.eval_to_const(operand)?; let operand = self.ecx.read_immediate(operand).discard_err()?; @@ -1007,11 +1005,16 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { location: Location, ) -> Option { match *operand { + Operand::RuntimeChecks(c) => { + Some(self.insert(self.tcx.types.bool, Value::RuntimeChecks(c))) + } Operand::Constant(ref constant) => Some(self.insert_constant(constant.const_)), Operand::Copy(ref mut place) | Operand::Move(ref mut place) => { let value = self.simplify_place_value(place, location)?; if let Some(const_) = self.try_as_constant(value) { *operand = Operand::Constant(Box::new(const_)); + } else if let Value::RuntimeChecks(c) = self.get(value) { + *operand = Operand::RuntimeChecks(c); } Some(value) } @@ -1034,7 +1037,6 @@ impl<'body, 'a, 'tcx> VnState<'body, 'a, 'tcx> { let op = self.simplify_operand(op, location)?; Value::Repeat(op, amount) } - Rvalue::NullaryOp(op) => Value::NullaryOp(op), Rvalue::Aggregate(..) => return self.simplify_aggregate(lhs, rvalue, location), Rvalue::Ref(_, borrow_kind, ref mut place) => { self.simplify_place_projection(place, location); @@ -1782,6 +1784,8 @@ impl<'tcx> VnState<'_, '_, 'tcx> { fn try_as_operand(&mut self, index: VnIndex, location: Location) -> Option> { if let Some(const_) = self.try_as_constant(index) { Some(Operand::Constant(Box::new(const_))) + } else if let Value::RuntimeChecks(c) = self.get(index) { + Some(Operand::RuntimeChecks(c)) } else if let Some(place) = self.try_as_place(index, location, false) { self.reused_locals.insert(place.local); Some(Operand::Copy(place)) diff --git a/compiler/rustc_mir_transform/src/instsimplify.rs b/compiler/rustc_mir_transform/src/instsimplify.rs index 4922d74743c9..fa9ceb018dd5 100644 --- a/compiler/rustc_mir_transform/src/instsimplify.rs +++ b/compiler/rustc_mir_transform/src/instsimplify.rs @@ -4,10 +4,11 @@ use rustc_abi::ExternAbi; use rustc_ast::attr; use rustc_hir::LangItem; use rustc_middle::bug; +use rustc_middle::mir::visit::MutVisitor; use rustc_middle::mir::*; use rustc_middle::ty::layout::ValidityRequirement; use rustc_middle::ty::{self, GenericArgsRef, Ty, TyCtxt, layout}; -use rustc_span::{DUMMY_SP, Symbol, sym}; +use rustc_span::{Symbol, sym}; use crate::simplify::simplify_duplicate_switch_targets; @@ -29,22 +30,22 @@ impl<'tcx> crate::MirPass<'tcx> for InstSimplify { } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let preserve_ub_checks = + attr::contains_name(tcx.hir_krate_attrs(), sym::rustc_preserve_ub_checks); + if !preserve_ub_checks { + SimplifyUbCheck { tcx }.visit_body(body); + } let ctx = InstSimplifyContext { tcx, local_decls: &body.local_decls, typing_env: body.typing_env(tcx), }; - let preserve_ub_checks = - attr::contains_name(tcx.hir_krate_attrs(), sym::rustc_preserve_ub_checks); for block in body.basic_blocks.as_mut() { for statement in block.statements.iter_mut() { let StatementKind::Assign(box (.., rvalue)) = &mut statement.kind else { continue; }; - if !preserve_ub_checks { - ctx.simplify_ub_check(rvalue); - } ctx.simplify_bool_cmp(rvalue); ctx.simplify_ref_deref(rvalue); ctx.simplify_ptr_aggregate(rvalue); @@ -168,17 +169,6 @@ impl<'tcx> InstSimplifyContext<'_, 'tcx> { } } - fn simplify_ub_check(&self, rvalue: &mut Rvalue<'tcx>) { - // FIXME: Should we do the same for overflow checks? - let Rvalue::NullaryOp(NullOp::RuntimeChecks(RuntimeChecks::UbChecks)) = *rvalue else { - return; - }; - - let const_ = Const::from_bool(self.tcx, self.tcx.sess.ub_checks()); - let constant = ConstOperand { span: DUMMY_SP, const_, user_ty: None }; - *rvalue = Rvalue::Use(Operand::Constant(Box::new(constant))); - } - fn simplify_cast(&self, rvalue: &mut Rvalue<'tcx>) { let Rvalue::Cast(kind, operand, cast_ty) = rvalue else { return }; @@ -362,3 +352,26 @@ fn resolve_rust_intrinsic<'tcx>( let intrinsic = tcx.intrinsic(def_id)?; Some((intrinsic.name, args)) } + +struct SimplifyUbCheck<'tcx> { + tcx: TyCtxt<'tcx>, +} + +impl<'tcx> MutVisitor<'tcx> for SimplifyUbCheck<'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_operand(&mut self, operand: &mut Operand<'tcx>, _: Location) { + if let Operand::RuntimeChecks(RuntimeChecks::UbChecks) = operand { + *operand = Operand::Constant(Box::new(ConstOperand { + span: rustc_span::DUMMY_SP, + user_ty: None, + const_: Const::Val( + ConstValue::from_bool(self.tcx.sess.ub_checks()), + self.tcx.types.bool, + ), + })); + } + } +} diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index c021e7d4c3ae..9e8de4926c6e 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -62,7 +62,9 @@ use rustc_middle::mir::interpret::Scalar; use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; use rustc_middle::ty::{self, ScalarInt, TyCtxt}; -use rustc_mir_dataflow::value_analysis::{Map, PlaceIndex, TrackElem, ValueIndex}; +use rustc_mir_dataflow::value_analysis::{ + Map, PlaceCollectionMode, PlaceIndex, TrackElem, ValueIndex, +}; use rustc_span::DUMMY_SP; use tracing::{debug, instrument, trace}; @@ -71,7 +73,6 @@ use crate::cost_checker::CostChecker; pub(super) struct JumpThreading; const MAX_COST: u8 = 100; -const MAX_PLACES: usize = 100; impl<'tcx> crate::MirPass<'tcx> for JumpThreading { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { @@ -95,7 +96,7 @@ impl<'tcx> crate::MirPass<'tcx> for JumpThreading { typing_env, ecx: InterpCx::new(tcx, DUMMY_SP, typing_env, DummyMachine), body, - map: Map::new(tcx, body, Some(MAX_PLACES)), + map: Map::new(tcx, body, PlaceCollectionMode::OnDemand), maybe_loop_headers: loops::maybe_loop_headers(body), entry_states: IndexVec::from_elem(ConditionSet::default(), &body.basic_blocks), }; @@ -273,6 +274,19 @@ impl ConditionSet { } impl<'a, 'tcx> TOFinder<'a, 'tcx> { + fn place(&mut self, place: Place<'tcx>, tail: Option) -> Option { + self.map.register_place(self.tcx, self.body, place, tail) + } + + fn value(&mut self, place: PlaceIndex) -> Option { + self.map.register_value(self.tcx, self.typing_env, place) + } + + fn place_value(&mut self, place: Place<'tcx>, tail: Option) -> Option { + let place = self.place(place, tail)?; + self.value(place) + } + /// Construct the condition set for `bb` from the terminator, without executing its effect. #[instrument(level = "trace", skip(self))] fn populate_from_outgoing_edges(&mut self, bb: BasicBlock) -> ConditionSet { @@ -397,7 +411,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { #[instrument(level = "trace", skip(self, state))] fn process_immediate(&mut self, lhs: PlaceIndex, rhs: ImmTy<'tcx>, state: &mut ConditionSet) { - if let Some(lhs) = self.map.value(lhs) + if let Some(lhs) = self.value(lhs) && let Immediate::Scalar(Scalar::Int(int)) = *rhs { state.fulfill_matches(lhs, int) @@ -412,10 +426,6 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { constant: OpTy<'tcx>, state: &mut ConditionSet, ) { - let values_inside = self.map.values_inside(lhs); - if !state.active.iter().any(|&(_, cond)| values_inside.contains(&cond.place)) { - return; - } self.map.for_each_projection_value( lhs, constant, @@ -450,9 +460,13 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { #[instrument(level = "trace", skip(self, state))] fn process_copy(&mut self, lhs: PlaceIndex, rhs: PlaceIndex, state: &mut ConditionSet) { let mut renames = FxHashMap::default(); - self.map.for_each_value_pair(rhs, lhs, &mut |rhs, lhs| { - renames.insert(lhs, rhs); - }); + self.map.register_copy_tree( + lhs, // tree to copy + rhs, // tree to build + &mut |lhs, rhs| { + renames.insert(lhs, rhs); + }, + ); state.for_each_mut(|c| { if let Some(rhs) = renames.get(&c.place) { c.place = *rhs @@ -474,9 +488,10 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { } // Transfer the conditions on the copied rhs. Operand::Move(rhs) | Operand::Copy(rhs) => { - let Some(rhs) = self.map.find(rhs.as_ref()) else { return }; + let Some(rhs) = self.place(*rhs, None) else { return }; self.process_copy(lhs, rhs, state) } + Operand::RuntimeChecks(_) => {} } } @@ -487,12 +502,12 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { rvalue: &Rvalue<'tcx>, state: &mut ConditionSet, ) { - let Some(lhs) = self.map.find(lhs_place.as_ref()) else { return }; + let Some(lhs) = self.place(*lhs_place, None) else { return }; match rvalue { Rvalue::Use(operand) => self.process_operand(lhs, operand, state), // Transfer the conditions on the copy rhs. Rvalue::Discriminant(rhs) => { - let Some(rhs) = self.map.find_discr(rhs.as_ref()) else { return }; + let Some(rhs) = self.place(*rhs, Some(TrackElem::Discriminant)) else { return }; self.process_copy(lhs, rhs, state) } // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. @@ -502,33 +517,37 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { // Do not support unions. AggregateKind::Adt(.., Some(_)) => return, AggregateKind::Adt(_, variant_index, ..) if agg_ty.is_enum() => { - if let Some(discr_target) = self.map.apply(lhs, TrackElem::Discriminant) - && let Some(discr_value) = self - .ecx - .discriminant_for_variant(agg_ty, *variant_index) - .discard_err() + let discr_ty = agg_ty.discriminant_ty(self.tcx); + let discr_target = + self.map.register_place_index(discr_ty, lhs, TrackElem::Discriminant); + if let Some(discr_value) = + self.ecx.discriminant_for_variant(agg_ty, *variant_index).discard_err() { self.process_immediate(discr_target, discr_value, state); } - if let Some(idx) = self.map.apply(lhs, TrackElem::Variant(*variant_index)) { - idx - } else { - return; - } + self.map.register_place_index( + agg_ty, + lhs, + TrackElem::Variant(*variant_index), + ) } _ => lhs, }; for (field_index, operand) in operands.iter_enumerated() { - if let Some(field) = self.map.apply(lhs, TrackElem::Field(field_index)) { - self.process_operand(field, operand, state); - } + let operand_ty = operand.ty(self.body, self.tcx); + let field = self.map.register_place_index( + operand_ty, + lhs, + TrackElem::Field(field_index), + ); + self.process_operand(field, operand, state); } } // Transfer the conditions on the copy rhs, after inverting the value of the condition. Rvalue::UnaryOp(UnOp::Not, Operand::Move(operand) | Operand::Copy(operand)) => { let layout = self.ecx.layout_of(operand.ty(self.body, self.tcx).ty).unwrap(); - let Some(lhs) = self.map.value(lhs) else { return }; - let Some(operand) = self.map.find_value(operand.as_ref()) else { return }; + let Some(lhs) = self.value(lhs) else { return }; + let Some(operand) = self.place_value(*operand, None) else { return }; state.retain_mut(|mut c| { if c.place == lhs { let value = self @@ -562,8 +581,8 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { // Avoid handling them, though this could be extended in the future. return; } - let Some(lhs) = self.map.value(lhs) else { return }; - let Some(operand) = self.map.find_value(operand.as_ref()) else { return }; + let Some(lhs) = self.value(lhs) else { return }; + let Some(operand) = self.place_value(*operand, None) else { return }; let Some(value) = value.const_.try_eval_scalar_int(self.tcx, self.typing_env) else { return; @@ -592,7 +611,9 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { // If we expect `discriminant(place) ?= A`, // we have an opportunity if `variant_index ?= A`. StatementKind::SetDiscriminant { box place, variant_index } => { - let Some(discr_target) = self.map.find_discr(place.as_ref()) else { return }; + let Some(discr_target) = self.place(*place, Some(TrackElem::Discriminant)) else { + return; + }; let enum_ty = place.ty(self.body, self.tcx).ty; // `SetDiscriminant` guarantees that the discriminant is now `variant_index`. // Even if the discriminant write does nothing due to niches, it is UB to set the @@ -608,7 +629,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume( Operand::Copy(place) | Operand::Move(place), )) => { - let Some(place) = self.map.find_value(place.as_ref()) else { return }; + let Some(place) = self.place_value(*place, None) else { return }; state.fulfill_matches(place, ScalarInt::TRUE); } StatementKind::Assign(box (lhs_place, rhs)) => { @@ -665,7 +686,7 @@ impl<'a, 'tcx> TOFinder<'a, 'tcx> { state: &mut ConditionSet, ) { let Some(discr) = discr.place() else { return }; - let Some(discr_idx) = self.map.find_value(discr.as_ref()) else { return }; + let Some(discr_idx) = self.place_value(discr, None) else { return }; let discr_ty = discr.ty(self.body, self.tcx).ty; let Ok(discr_layout) = self.ecx.layout_of(discr_ty) else { return }; diff --git a/compiler/rustc_mir_transform/src/known_panics_lint.rs b/compiler/rustc_mir_transform/src/known_panics_lint.rs index 1f5d31932f1a..caaf300a88d6 100644 --- a/compiler/rustc_mir_transform/src/known_panics_lint.rs +++ b/compiler/rustc_mir_transform/src/known_panics_lint.rs @@ -282,6 +282,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { /// or `eval_place`, depending on the variant of `Operand` used. fn eval_operand(&mut self, op: &Operand<'tcx>) -> Option> { match *op { + Operand::RuntimeChecks(_) => None, Operand::Constant(ref c) => self.eval_constant(c), Operand::Move(place) | Operand::Copy(place) => self.eval_place(place), } @@ -444,7 +445,6 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { | Rvalue::Cast(..) | Rvalue::ShallowInitBox(..) | Rvalue::Discriminant(..) - | Rvalue::NullaryOp(..) | Rvalue::WrapUnsafeBinder(..) => {} } @@ -605,8 +605,6 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { Ref(..) | RawPtr(..) => return None, - NullaryOp(NullOp::RuntimeChecks(_)) => return None, - ShallowInitBox(..) => return None, Cast(ref kind, ref value, to) => match kind { diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 6415f3908499..701d7ff854a7 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -261,7 +261,7 @@ fn remap_mir_for_const_eval_select<'tcx>( if context == hir::Constness::Const { called_in_const } else { called_at_rt }; let (method, place): (fn(Place<'tcx>) -> Operand<'tcx>, Place<'tcx>) = match tupled_args.node { - Operand::Constant(_) => { + Operand::Constant(_) | Operand::RuntimeChecks(_) => { // There is no good way of extracting a tuple arg from a constant // (const generic stuff) so we just create a temporary and deconstruct // that. diff --git a/compiler/rustc_mir_transform/src/lint.rs b/compiler/rustc_mir_transform/src/lint.rs index 2ab49645dc44..88297a4efef7 100644 --- a/compiler/rustc_mir_transform/src/lint.rs +++ b/compiler/rustc_mir_transform/src/lint.rs @@ -88,7 +88,6 @@ impl<'a, 'tcx> Visitor<'tcx> for Lint<'a, 'tcx> { | Rvalue::ShallowInitBox(..) | Rvalue::WrapUnsafeBinder(..) => true, Rvalue::ThreadLocalRef(..) - | Rvalue::NullaryOp(..) | Rvalue::UnaryOp(..) | Rvalue::BinaryOp(..) | Rvalue::Ref(..) diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index f25874fbbcb8..dcee54c37108 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -35,7 +35,7 @@ impl<'tcx> crate::MirPass<'tcx> for LowerIntrinsics { terminator.source_info, StatementKind::Assign(Box::new(( *destination, - Rvalue::NullaryOp(NullOp::RuntimeChecks(op)), + Rvalue::Use(Operand::RuntimeChecks(op)), ))), )); terminator.kind = TerminatorKind::Goto { target }; diff --git a/compiler/rustc_mir_transform/src/promote_consts.rs b/compiler/rustc_mir_transform/src/promote_consts.rs index 7d631e96c32a..11266ccc2832 100644 --- a/compiler/rustc_mir_transform/src/promote_consts.rs +++ b/compiler/rustc_mir_transform/src/promote_consts.rs @@ -360,6 +360,10 @@ impl<'tcx> Validator<'_, 'tcx> { match operand { Operand::Copy(place) | Operand::Move(place) => self.validate_place(place.as_ref()), + // `RuntimeChecks` behaves different in const-eval and runtime MIR, + // so we do not promote it. + Operand::RuntimeChecks(_) => Err(Unpromotable), + // The qualifs for a constant (e.g. `HasMutInterior`) are checked in // `validate_rvalue` upon access. Operand::Constant(c) => { @@ -443,10 +447,6 @@ impl<'tcx> Validator<'_, 'tcx> { self.validate_operand(operand)?; } - Rvalue::NullaryOp(op) => match op { - NullOp::RuntimeChecks(_) => {} - }, - Rvalue::ShallowInitBox(_, _) => return Err(Unpromotable), Rvalue::UnaryOp(op, operand) => { diff --git a/compiler/rustc_mir_transform/src/simplify_branches.rs b/compiler/rustc_mir_transform/src/simplify_branches.rs index ba2286fd40a7..70ae3433353b 100644 --- a/compiler/rustc_mir_transform/src/simplify_branches.rs +++ b/compiler/rustc_mir_transform/src/simplify_branches.rs @@ -41,7 +41,7 @@ impl<'tcx> crate::MirPass<'tcx> for SimplifyConstCondition { { Some(const_operand) } - Operand::Copy(_) | Operand::Move(_) => None, + Operand::Copy(_) | Operand::Move(_) | Operand::RuntimeChecks(_) => None, } } diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs index 801383493837..a6ed66c8427b 100644 --- a/compiler/rustc_mir_transform/src/sroa.rs +++ b/compiler/rustc_mir_transform/src/sroa.rs @@ -392,11 +392,14 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> { // a_1 = move? place.1 // ... // ``` - StatementKind::Assign(box (lhs, Rvalue::Use(ref op))) => { - let (rplace, copy) = match *op { - Operand::Copy(rplace) => (rplace, true), - Operand::Move(rplace) => (rplace, false), - Operand::Constant(_) => bug!(), + StatementKind::Assign(box ( + lhs, + Rvalue::Use(ref op @ (Operand::Copy(rplace) | Operand::Move(rplace))), + )) => { + let copy = match *op { + Operand::Copy(_) => true, + Operand::Move(_) => false, + Operand::Constant(_) | Operand::RuntimeChecks(_) => bug!(), }; if let Some(final_locals) = self.replacements.place_fragments(lhs) { for (field, ty, new_local) in final_locals { diff --git a/compiler/rustc_mir_transform/src/validate.rs b/compiler/rustc_mir_transform/src/validate.rs index cf8247c12abd..1afdb4639a0c 100644 --- a/compiler/rustc_mir_transform/src/validate.rs +++ b/compiler/rustc_mir_transform/src/validate.rs @@ -1439,7 +1439,6 @@ impl<'a, 'tcx> Visitor<'tcx> for TypeChecker<'a, 'tcx> { Rvalue::Repeat(_, _) | Rvalue::ThreadLocalRef(_) | Rvalue::RawPtr(_, _) - | Rvalue::NullaryOp(NullOp::RuntimeChecks(_)) | Rvalue::Discriminant(_) => {} Rvalue::WrapUnsafeBinder(op, ty) => { diff --git a/compiler/rustc_monomorphize/src/collector.rs b/compiler/rustc_monomorphize/src/collector.rs index cd699436e012..4b2f8e03afc1 100644 --- a/compiler/rustc_monomorphize/src/collector.rs +++ b/compiler/rustc_monomorphize/src/collector.rs @@ -1582,7 +1582,7 @@ impl<'v> RootCollector<'_, 'v> { } fn process_impl_item(&mut self, id: hir::ImplItemId) { - if matches!(self.tcx.def_kind(id.owner_id), DefKind::AssocFn) { + if self.tcx.def_kind(id.owner_id) == DefKind::AssocFn { self.push_if_root(id.owner_id.def_id); } } @@ -1720,7 +1720,7 @@ fn create_mono_items_for_default_impls<'tcx>( ) { let impl_ = tcx.impl_trait_header(item.owner_id); - if matches!(impl_.polarity, ty::ImplPolarity::Negative) { + if impl_.polarity == ty::ImplPolarity::Negative { return; } diff --git a/compiler/rustc_parse/Cargo.toml b/compiler/rustc_parse/Cargo.toml index 5239c6290bfb..04a51c905661 100644 --- a/compiler/rustc_parse/Cargo.toml +++ b/compiler/rustc_parse/Cargo.toml @@ -20,8 +20,8 @@ rustc_session = { path = "../rustc_session" } rustc_span = { path = "../rustc_span" } thin-vec = "0.2.12" tracing = "0.1" -unicode-normalization = "0.1.11" -unicode-width = "0.2.0" +unicode-normalization = "0.1.25" +unicode-width = "0.2.2" # tidy-alphabetical-end [dev-dependencies] diff --git a/compiler/rustc_parse/messages.ftl b/compiler/rustc_parse/messages.ftl index 87d1173c0d48..747895c80469 100644 --- a/compiler/rustc_parse/messages.ftl +++ b/compiler/rustc_parse/messages.ftl @@ -162,6 +162,8 @@ parse_default_not_followed_by_item = `default` is not followed by an item .label = the `default` qualifier .note = only `fn`, `const`, `type`, or `impl` items may be prefixed by `default` +parse_delegation_non_trait_impl_reuse = only trait impls can be reused + parse_do_catch_syntax_removed = found removed `do catch` syntax .note = following RFC #2388, the new non-placeholder syntax is `try` .suggestion = replace with the new syntax diff --git a/compiler/rustc_parse/src/errors.rs b/compiler/rustc_parse/src/errors.rs index 698d8f76aaa6..3b72c9802afd 100644 --- a/compiler/rustc_parse/src/errors.rs +++ b/compiler/rustc_parse/src/errors.rs @@ -3671,3 +3671,10 @@ pub(crate) struct VarargsWithoutPattern { #[suggestion(code = "_: ...", applicability = "machine-applicable")] pub span: Span, } + +#[derive(Diagnostic)] +#[diag(parse_delegation_non_trait_impl_reuse)] +pub(crate) struct ImplReuseInherentImpl { + #[primary_span] + pub span: Span, +} diff --git a/compiler/rustc_parse/src/lexer/tokentrees.rs b/compiler/rustc_parse/src/lexer/tokentrees.rs index 634f4c30b260..f3415aa47d33 100644 --- a/compiler/rustc_parse/src/lexer/tokentrees.rs +++ b/compiler/rustc_parse/src/lexer/tokentrees.rs @@ -117,7 +117,7 @@ impl<'psess, 'src> Lexer<'psess, 'src> { candidate = Some(*delimiter_span); } } - let (_, _) = self.diag_info.open_delimiters.pop().unwrap(); + self.diag_info.open_delimiters.pop().unwrap(); self.diag_info.unmatched_delims.push(UnmatchedDelim { found_delim: Some(close_delim), found_span: self.token.span, diff --git a/compiler/rustc_parse/src/lib.rs b/compiler/rustc_parse/src/lib.rs index df8f970e0599..1fb44df65347 100644 --- a/compiler/rustc_parse/src/lib.rs +++ b/compiler/rustc_parse/src/lib.rs @@ -22,10 +22,10 @@ use rustc_ast::token; use rustc_ast::tokenstream::TokenStream; use rustc_ast_pretty::pprust; use rustc_errors::{Diag, EmissionGuarantee, FatalError, PResult, pluralize}; +pub use rustc_lexer::UNICODE_VERSION; use rustc_session::parse::ParseSess; use rustc_span::source_map::SourceMap; use rustc_span::{FileName, SourceFile, Span}; -pub use unicode_normalization::UNICODE_VERSION as UNICODE_NORMALIZATION_VERSION; pub const MACRO_ARGUMENTS: Option<&str> = Some("macro arguments"); @@ -39,6 +39,44 @@ pub mod lexer; mod errors; +// Make sure that the Unicode version of the dependencies is the same. +const _: () = { + let rustc_lexer = rustc_lexer::UNICODE_VERSION; + let rustc_span = rustc_span::UNICODE_VERSION; + let normalization = unicode_normalization::UNICODE_VERSION; + let width = unicode_width::UNICODE_VERSION; + + if rustc_lexer.0 != rustc_span.0 + || rustc_lexer.1 != rustc_span.1 + || rustc_lexer.2 != rustc_span.2 + { + panic!( + "rustc_lexer and rustc_span must use the same Unicode version, \ + `rustc_lexer::UNICODE_VERSION` and `rustc_span::UNICODE_VERSION` are \ + different." + ); + } + + if rustc_lexer.0 != normalization.0 + || rustc_lexer.1 != normalization.1 + || rustc_lexer.2 != normalization.2 + { + panic!( + "rustc_lexer and unicode-normalization must use the same Unicode version, \ + `rustc_lexer::UNICODE_VERSION` and `unicode_normalization::UNICODE_VERSION` are \ + different." + ); + } + + if rustc_lexer.0 != width.0 || rustc_lexer.1 != width.1 || rustc_lexer.2 != width.2 { + panic!( + "rustc_lexer and unicode-width must use the same Unicode version, \ + `rustc_lexer::UNICODE_VERSION` and `unicode_width::UNICODE_VERSION` are \ + different." + ); + } +}; + rustc_fluent_macro::fluent_messages! { "../messages.ftl" } // Unwrap the result if `Ok`, otherwise emit the diagnostics and abort. diff --git a/compiler/rustc_parse/src/parser/asm.rs b/compiler/rustc_parse/src/parser/asm.rs index caec877232a6..3fab234adaad 100644 --- a/compiler/rustc_parse/src/parser/asm.rs +++ b/compiler/rustc_parse/src/parser/asm.rs @@ -77,7 +77,7 @@ fn eat_operand_keyword<'a>( exp: ExpKeywordPair, asm_macro: AsmMacro, ) -> PResult<'a, bool> { - if matches!(asm_macro, AsmMacro::Asm) { + if asm_macro == AsmMacro::Asm { Ok(p.eat_keyword(exp)) } else { let span = p.token.span; diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 3f0853a3c54d..8bbf534294f4 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -1521,7 +1521,7 @@ impl<'a> Parser<'a> { }, ) } else if this.check_inline_const(0) { - this.parse_const_block(lo, false) + this.parse_const_block(lo) } else if this.may_recover() && this.is_do_catch_block() { this.recover_do_catch() } else if this.is_try_block() { @@ -3100,7 +3100,7 @@ impl<'a> Parser<'a> { pub(crate) fn eat_label(&mut self) -> Option