user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #4350 from rust-lang/rustup-2025-05-27

Browse source 
Automatic Rustup
This commit is contained in:
Ralf Jung 2025-05-27 06:44:50 +00:00 • committed by GitHub
commit 467591f99c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
275 changed files with 3623 additions and 2750 deletions
Download patch file Download diff file Expand all files Collapse all files

238
compiler/rustc_ast/src/mut_visit.rs
View file

@ -20,7 +20,7 @@ use thin_vec::ThinVec;
use crate::ast::*;
use crate::ptr::P;
use crate::tokenstream::*;
use crate::visit::{AssocCtxt, BoundKind, FnCtxt, try_visit};
use crate::visit::{AssocCtxt, BoundKind, FnCtxt, try_visit, visit_opt, walk_list};
pub trait ExpectOne<A: Array> {
fn expect_one(self, err: &'static str) -> A::Item;
@ -33,18 +33,6 @@ impl<A: Array> ExpectOne<A> for SmallVec<A> {
}
}
pub trait WalkItemKind {
type Ctxt;
fn walk(
&mut self,
span: Span,
id: NodeId,
visibility: &mut Visibility,
ctxt: Self::Ctxt,
visitor: &mut impl MutVisitor,
);
}
pub trait MutVisitor: Sized {
// Methods in this trait have one of three forms:
//
@ -451,11 +439,6 @@ fn visit_thin_exprs<T: MutVisitor>(vis: &mut T, exprs: &mut ThinVec<P<Expr>>) {
exprs.flat_map_in_place(|expr| vis.filter_map_expr(expr))
}
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
fn visit_bounds<T: MutVisitor>(vis: &mut T, bounds: &mut GenericBounds, ctxt: BoundKind) {
visit_vec(bounds, |bound| vis.visit_param_bound(bound, ctxt));
}
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
fn visit_attr_args<T: MutVisitor>(vis: &mut T, args: &mut AttrArgs) {
match args {
@ -610,12 +593,6 @@ pub fn walk_ty_pat<T: MutVisitor>(vis: &mut T, ty: &mut P<TyPat>) {
vis.visit_span(span);
}
fn walk_foreign_mod<T: MutVisitor>(vis: &mut T, foreign_mod: &mut ForeignMod) {
let ForeignMod { extern_span: _, safety, abi: _, items } = foreign_mod;
visit_safety(vis, safety);
items.flat_map_in_place(|item| vis.flat_map_foreign_item(item));
}
pub fn walk_variant<T: MutVisitor>(visitor: &mut T, variant: &mut Variant) {
let Variant { ident, vis, attrs, id, data, disr_expr, span, is_placeholder: _ } = variant;
visitor.visit_id(id);
@ -771,22 +748,6 @@ pub fn walk_flat_map_param<T: MutVisitor>(vis: &mut T, mut param: Param) -> Smal
smallvec![param]
}
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
fn visit_defaultness<T: MutVisitor>(vis: &mut T, defaultness: &mut Defaultness) {
match defaultness {
Defaultness::Default(span) => vis.visit_span(span),
Defaultness::Final => {}
}
}
// No `noop_` prefix because there isn't a corresponding method in `MutVisitor`.
fn visit_polarity<T: MutVisitor>(vis: &mut T, polarity: &mut ImplPolarity) {
match polarity {
ImplPolarity::Positive => {}
ImplPolarity::Negative(span) => vis.visit_span(span),
}
}
fn walk_closure_binder<T: MutVisitor>(vis: &mut T, binder: &mut ClosureBinder) {
match binder {
ClosureBinder::NotPresent => {}
@ -1080,169 +1041,15 @@ pub fn walk_item_kind<K: WalkItemKind>(
kind.walk(span, id, visibility, ctxt, vis)
}
impl WalkItemKind for ItemKind {
type Ctxt = ();
fn walk(
&mut self,
span: Span,
id: NodeId,
visibility: &mut Visibility,
_ctxt: Self::Ctxt,
vis: &mut impl MutVisitor,
) {
match self {
ItemKind::ExternCrate(_orig_name, ident) => vis.visit_ident(ident),
ItemKind::Use(use_tree) => vis.visit_use_tree(use_tree),
ItemKind::Static(box StaticItem {
ident,
ty,
safety: _,
mutability: _,
expr,
define_opaque,
}) => {
vis.visit_ident(ident);
vis.visit_ty(ty);
visit_opt(expr, |expr| vis.visit_expr(expr));
walk_define_opaques(vis, define_opaque);
}
ItemKind::Const(item) => {
walk_const_item(vis, item);
}
ItemKind::Fn(func) => {
vis.visit_fn(FnKind::Fn(FnCtxt::Free, visibility, &mut *func), span, id);
}
ItemKind::Mod(safety, ident, mod_kind) => {
visit_safety(vis, safety);
vis.visit_ident(ident);
match mod_kind {
ModKind::Loaded(
items,
_inline,
ModSpans { inner_span, inject_use_span },
_,
) => {
items.flat_map_in_place(|item| vis.flat_map_item(item));
vis.visit_span(inner_span);
vis.visit_span(inject_use_span);
}
ModKind::Unloaded => {}
}
}
ItemKind::ForeignMod(nm) => vis.visit_foreign_mod(nm),
ItemKind::GlobalAsm(asm) => vis.visit_inline_asm(asm),
ItemKind::TyAlias(box TyAlias {
defaultness,
ident,
generics,
where_clauses,
bounds,
ty,
}) => {
visit_defaultness(vis, defaultness);
vis.visit_ident(ident);
vis.visit_generics(generics);
visit_bounds(vis, bounds, BoundKind::Bound);
visit_opt(ty, |ty| vis.visit_ty(ty));
walk_ty_alias_where_clauses(vis, where_clauses);
}
ItemKind::Enum(ident, EnumDef { variants }, generics) => {
vis.visit_ident(ident);
vis.visit_generics(generics);
variants.flat_map_in_place(|variant| vis.flat_map_variant(variant));
}
ItemKind::Struct(ident, variant_data, generics)
| ItemKind::Union(ident, variant_data, generics) => {
vis.visit_ident(ident);
vis.visit_generics(generics);
vis.visit_variant_data(variant_data);
}
ItemKind::Impl(box Impl {
defaultness,
safety,
generics,
constness,
polarity,
of_trait,
self_ty,
items,
}) => {
visit_defaultness(vis, defaultness);
visit_safety(vis, safety);
vis.visit_generics(generics);
visit_constness(vis, constness);
visit_polarity(vis, polarity);
visit_opt(of_trait, |trait_ref| vis.visit_trait_ref(trait_ref));
vis.visit_ty(self_ty);
items.flat_map_in_place(|item| {
vis.flat_map_assoc_item(item, AssocCtxt::Impl { of_trait: of_trait.is_some() })
});
}
ItemKind::Trait(box Trait { safety, is_auto: _, ident, generics, bounds, items }) => {
visit_safety(vis, safety);
vis.visit_ident(ident);
vis.visit_generics(generics);
visit_bounds(vis, bounds, BoundKind::Bound);
items.flat_map_in_place(|item| vis.flat_map_assoc_item(item, AssocCtxt::Trait));
}
ItemKind::TraitAlias(ident, generics, bounds) => {
vis.visit_ident(ident);
vis.visit_generics(generics);
visit_bounds(vis, bounds, BoundKind::Bound);
}
ItemKind::MacCall(m) => vis.visit_mac_call(m),
ItemKind::MacroDef(ident, def) => {
vis.visit_ident(ident);
vis.visit_macro_def(def)
}
ItemKind::Delegation(box Delegation {
id,
qself,
path,
ident,
rename,
body,
from_glob: _,
}) => {
vis.visit_id(id);
vis.visit_qself(qself);
vis.visit_path(path);
vis.visit_ident(ident);
if let Some(rename) = rename {
vis.visit_ident(rename);
}
if let Some(body) = body {
vis.visit_block(body);
}
}
ItemKind::DelegationMac(box DelegationMac { qself, prefix, suffixes, body }) => {
vis.visit_qself(qself);
vis.visit_path(prefix);
if let Some(suffixes) = suffixes {
for (ident, rename) in suffixes {
vis.visit_ident(ident);
if let Some(rename) = rename {
vis.visit_ident(rename);
}
}
}
if let Some(body) = body {
vis.visit_block(body);
}
}
}
}
}
impl WalkItemKind for AssocItemKind {
type Ctxt = AssocCtxt;
fn walk(
fn walk<V: MutVisitor>(
&mut self,
span: Span,
id: NodeId,
visibility: &mut Visibility,
ctxt: Self::Ctxt,
visitor: &mut impl MutVisitor,
visitor: &mut V,
) {
match self {
AssocItemKind::Const(item) => {
@ -1306,16 +1113,6 @@ impl WalkItemKind for AssocItemKind {
}
}
fn walk_const_item<T: MutVisitor>(vis: &mut T, item: &mut ConstItem) {
let ConstItem { defaultness, ident, generics, ty, expr, define_opaque } = item;
visit_defaultness(vis, defaultness);
vis.visit_ident(ident);
vis.visit_generics(generics);
vis.visit_ty(ty);
visit_opt(expr, |expr| vis.visit_expr(expr));
walk_define_opaques(vis, define_opaque);
}
pub fn walk_crate<T: MutVisitor>(vis: &mut T, krate: &mut Crate) {
let Crate { attrs, items, spans, id, is_placeholder: _ } = krate;
vis.visit_id(id);
@ -1334,19 +1131,6 @@ pub fn walk_assoc_item(visitor: &mut impl MutVisitor, item: &mut P<AssocItem>, c
walk_item_ctxt(visitor, item, ctxt)
}
fn walk_item_ctxt<K: WalkItemKind>(
visitor: &mut impl MutVisitor,
item: &mut P<Item<K>>,
ctxt: K::Ctxt,
) {
let Item { attrs, id, kind, vis, span, tokens: _ } = item.deref_mut();
visitor.visit_id(id);
visit_attrs(visitor, attrs);
visitor.visit_vis(vis);
kind.walk(*span, *id, vis, ctxt, visitor);
visitor.visit_span(span);
}
pub fn walk_flat_map_item(vis: &mut impl MutVisitor, mut item: P<Item>) -> SmallVec<[P<Item>; 1]> {
vis.visit_item(&mut item);
smallvec![item]
@ -1371,13 +1155,13 @@ pub fn walk_flat_map_assoc_item(
impl WalkItemKind for ForeignItemKind {
type Ctxt = ();
fn walk(
fn walk<V: MutVisitor>(
&mut self,
span: Span,
id: NodeId,
visibility: &mut Visibility,
_ctxt: Self::Ctxt,
visitor: &mut impl MutVisitor,
visitor: &mut V,
) {
match self {
ForeignItemKind::Static(box StaticItem {
@ -1786,18 +1570,6 @@ fn walk_capture_by<T: MutVisitor>(vis: &mut T, capture_by: &mut CaptureBy) {
}
}
fn walk_define_opaques<T: MutVisitor>(
vis: &mut T,
define_opaque: &mut Option<ThinVec<(NodeId, Path)>>,
) {
if let Some(define_opaque) = define_opaque {
for (id, path) in define_opaque {
vis.visit_id(id);
vis.visit_path(path)
}
}
}
/// Some value for the AST node that is valid but possibly meaningless. Similar
/// to `Default` but not intended for wide use. The value will never be used
/// meaningfully, it exists just to support unwinding in `visit_clobber` in the

475
compiler/rustc_ast/src/visit.rs
View file

@ -112,18 +112,6 @@ pub enum LifetimeCtxt {
GenericArg,
}
pub trait WalkItemKind {
type Ctxt;
fn walk<'a, V: Visitor<'a>>(
&'a self,
span: Span,
id: NodeId,
visibility: &'a Visibility,
ctxt: Self::Ctxt,
visitor: &mut V,
) -> V::Result;
}
/// Each method of the `Visitor` trait is a hook to be potentially
/// overridden. Each method's default implementation recursively visits
/// the substructure of the input via the corresponding `walk` method;
@ -141,6 +129,9 @@ pub trait Visitor<'ast>: Sized {
fn visit_ident(&mut self, _ident: &'ast Ident) -> Self::Result {
Self::Result::output()
}
fn visit_foreign_mod(&mut self, nm: &'ast ForeignMod) -> Self::Result {
walk_foreign_mod(self, nm)
}
fn visit_foreign_item(&mut self, i: &'ast ForeignItem) -> Self::Result {
walk_item(self, i)
}
@ -242,7 +233,7 @@ pub trait Visitor<'ast>: Sized {
fn visit_mac_call(&mut self, mac: &'ast MacCall) -> Self::Result {
walk_mac(self, mac)
}
fn visit_mac_def(&mut self, _mac: &'ast MacroDef, _id: NodeId) -> Self::Result {
fn visit_macro_def(&mut self, _mac: &'ast MacroDef, _id: NodeId) -> Self::Result {
Self::Result::output()
}
fn visit_path(&mut self, path: &'ast Path, _id: NodeId) -> Self::Result {
@ -318,6 +309,18 @@ pub trait Visitor<'ast>: Sized {
#[macro_export]
macro_rules! common_visitor_and_walkers {
($(($mut: ident))? $Visitor:ident$(<$lt:lifetime>)?) => {
pub trait WalkItemKind {
type Ctxt;
fn walk<$($lt,)? V: $Visitor$(<$lt>)?>(
&$($lt)? $($mut)? self,
span: Span,
id: NodeId,
visibility: &$($lt)? $($mut)? Visibility,
ctxt: Self::Ctxt,
visitor: &mut V,
) $(-> <V as Visitor<$lt>>::Result)?;
}
// this is only used by the MutVisitor. We include this symmetry here to make writing other functions easier
$(${ignore($lt)}
#[expect(unused, rustc::pass_by_value)]
@ -325,7 +328,7 @@ macro_rules! common_visitor_and_walkers {
)?
fn visit_span<$($lt,)? V: $Visitor$(<$lt>)?>(visitor: &mut V, span: &$($lt)? $($mut)? Span) $(-> <V as Visitor<$lt>>::Result)? {
$(
let _ = stringify!($mut);
${ignore($mut)}
visitor.visit_span(span);
)?
$(${ignore($lt)}V::Result::output())?
@ -338,7 +341,7 @@ macro_rules! common_visitor_and_walkers {
)?
fn visit_id<$($lt,)? V: $Visitor$(<$lt>)?>(visitor: &mut V, id: &$($lt)? $($mut)? NodeId) $(-> <V as Visitor<$lt>>::Result)? {
$(
let _ = stringify!($mut);
${ignore($mut)}
visitor.visit_id(id);
)?
$(${ignore($lt)}V::Result::output())?
@ -362,6 +365,27 @@ macro_rules! common_visitor_and_walkers {
}
}
fn visit_defaultness<$($lt,)? V: $Visitor$(<$lt>)?>(vis: &mut V, defaultness: &$($lt)? $($mut)? Defaultness) $(-> <V as Visitor<$lt>>::Result)? {
match defaultness {
Defaultness::Default(span) => visit_span(vis, span),
Defaultness::Final => {
$(<V as Visitor<$lt>>::Result::output())?
}
}
}
fn visit_polarity<$($lt,)? V: $Visitor$(<$lt>)?>(vis: &mut V, polarity: &$($lt)? $($mut)? ImplPolarity) $(-> <V as Visitor<$lt>>::Result)? {
match polarity {
ImplPolarity::Positive => { $(<V as Visitor<$lt>>::Result::output())? }
ImplPolarity::Negative(span) => visit_span(vis, span),
}
}
fn visit_bounds<$($lt,)? V: $Visitor$(<$lt>)?>(visitor: &mut V, bounds: &$($lt)? $($mut)? GenericBounds, ctxt: BoundKind) $(-> <V as Visitor<$lt>>::Result)? {
walk_list!(visitor, visit_param_bound, bounds, ctxt);
$(<V as Visitor<$lt>>::Result::output())?
}
pub fn walk_label<$($lt,)? V: $Visitor$(<$lt>)?>(visitor: &mut V, Label { ident }: &$($lt)? $($mut)? Label) $(-> <V as Visitor<$lt>>::Result)? {
visitor.visit_ident(ident)
}
@ -379,6 +403,246 @@ macro_rules! common_visitor_and_walkers {
try_visit!(visit_id(visitor, id));
visitor.visit_ident(ident)
}
fn walk_item_ctxt<$($lt,)? V: $Visitor$(<$lt>)?, K: WalkItemKind>(
visitor: &mut V,
item: &$($mut P<Item<K>>)? $($lt Item<K>)?,
ctxt: K::Ctxt,
) $(-> <V as Visitor<$lt>>::Result)? {
let Item { attrs, id, kind, vis, span, tokens: _ } = &$($mut *)? *item;
try_visit!(visit_id(visitor, id));
walk_list!(visitor, visit_attribute, attrs);
try_visit!(visitor.visit_vis(vis));
try_visit!(kind.walk(*span, *id, vis, ctxt, visitor));
visit_span(visitor, span)
}
impl WalkItemKind for ItemKind {
type Ctxt = ();
fn walk<$($lt,)? V: $Visitor$(<$lt>)?>(
&$($lt)? $($mut)? self,
span: Span,
id: NodeId,
visibility: &$($lt)? $($mut)? Visibility,
_ctxt: Self::Ctxt,
vis: &mut V,
) $(-> <V as Visitor<$lt>>::Result)? {
match self {
ItemKind::ExternCrate(_orig_name, ident) => vis.visit_ident(ident),
// FIXME(fee1-dead): look into this weird assymetry
ItemKind::Use(use_tree) => vis.visit_use_tree(use_tree$(${ignore($lt)}, id, false)?),
ItemKind::Static(box StaticItem {
ident,
ty,
safety: _,
mutability: _,
expr,
define_opaque,
}) => {
try_visit!(vis.visit_ident(ident));
try_visit!(vis.visit_ty(ty));
visit_opt!(vis, visit_expr, expr);
walk_define_opaques(vis, define_opaque)
}
ItemKind::Const(item) => {
walk_const_item(vis, item)
}
ItemKind::Fn(func) => {
let kind = FnKind::Fn(FnCtxt::Free, visibility, &$($mut)? *func);
vis.visit_fn(kind, span, id)
}
ItemKind::Mod(safety, ident, mod_kind) => {
try_visit!(visit_safety(vis, safety));
try_visit!(vis.visit_ident(ident));
match mod_kind {
ModKind::Loaded(
items,
_inline,
ModSpans { inner_span, inject_use_span },
_,
) => {
$(${ignore($mut)}
items.flat_map_in_place(|item| vis.flat_map_item(item));
)?
$(${ignore($lt)}
walk_list!(vis, visit_item, items);
)?
try_visit!(visit_span(vis, inner_span));
try_visit!(visit_span(vis, inject_use_span));
}
ModKind::Unloaded => {}
}
$(<V as Visitor<$lt>>::Result::output())?
}
ItemKind::ForeignMod(nm) => vis.visit_foreign_mod(nm),
ItemKind::GlobalAsm(asm) => vis.visit_inline_asm(asm),
ItemKind::TyAlias(box TyAlias {
defaultness,
ident,
generics,
$(${ignore($lt)} #[expect(unused)])?
where_clauses,
bounds,
ty,
}) => {
try_visit!(visit_defaultness(vis, defaultness));
try_visit!(vis.visit_ident(ident));
try_visit!(vis.visit_generics(generics));
try_visit!(visit_bounds(vis, bounds, BoundKind::Bound));
visit_opt!(vis, visit_ty, ty);
$(${ignore($mut)}
walk_ty_alias_where_clauses(vis, where_clauses);
)?
$(<V as Visitor<$lt>>::Result::output())?
}
ItemKind::Enum(ident, enum_definition, generics) => {
try_visit!(vis.visit_ident(ident));
try_visit!(vis.visit_generics(generics));
$(${ignore($mut)}
enum_definition.variants.flat_map_in_place(|variant| vis.flat_map_variant(variant));
)?
$(${ignore($lt)}vis.visit_enum_def(enum_definition))?
}
ItemKind::Struct(ident, variant_data, generics)
| ItemKind::Union(ident, variant_data, generics) => {
try_visit!(vis.visit_ident(ident));
try_visit!(vis.visit_generics(generics));
vis.visit_variant_data(variant_data)
}
ItemKind::Impl(box Impl {
defaultness,
safety,
generics,
constness,
polarity,
of_trait,
self_ty,
items,
}) => {
try_visit!(visit_defaultness(vis, defaultness));
try_visit!(visit_safety(vis, safety));
try_visit!(vis.visit_generics(generics));
try_visit!(visit_constness(vis, constness));
try_visit!(visit_polarity(vis, polarity));
visit_opt!(vis, visit_trait_ref, of_trait);
try_visit!(vis.visit_ty(self_ty));
$(${ignore($mut)}
items.flat_map_in_place(|item| {
vis.flat_map_assoc_item(item, AssocCtxt::Impl { of_trait: of_trait.is_some() })
});
)?
$(${ignore($lt)}
walk_list!(
vis,
visit_assoc_item,
items,
AssocCtxt::Impl { of_trait: of_trait.is_some() }
);
<V as Visitor<$lt>>::Result::output()
)?
}
ItemKind::Trait(box Trait { safety, is_auto: _, ident, generics, bounds, items }) => {
try_visit!(visit_safety(vis, safety));
try_visit!(vis.visit_ident(ident));
try_visit!(vis.visit_generics(generics));
try_visit!(visit_bounds(vis, bounds, BoundKind::Bound));
$(${ignore($mut)}
items.flat_map_in_place(|item| {
vis.flat_map_assoc_item(item, AssocCtxt::Trait)
});
)?
$(${ignore($lt)}
walk_list!(vis, visit_assoc_item, items, AssocCtxt::Trait);
<V as Visitor<$lt>>::Result::output()
)?
}
ItemKind::TraitAlias(ident, generics, bounds) => {
try_visit!(vis.visit_ident(ident));
try_visit!(vis.visit_generics(generics));
visit_bounds(vis, bounds, BoundKind::Bound)
}
ItemKind::MacCall(m) => vis.visit_mac_call(m),
ItemKind::MacroDef(ident, def) => {
try_visit!(vis.visit_ident(ident));
// FIXME(fee1-dead) assymetry
vis.visit_macro_def(def$(${ignore($lt)}, id)?)
}
ItemKind::Delegation(box Delegation {
id,
qself,
path,
ident,
rename,
body,
from_glob: _,
}) => {
try_visit!(visit_id(vis, id));
try_visit!(vis.visit_qself(qself));
try_visit!(vis.visit_path(path$(${ignore($lt)}, *id)?));
try_visit!(vis.visit_ident(ident));
if let Some(rename) = rename {
try_visit!(vis.visit_ident(rename));
}
if let Some(body) = body {
try_visit!(vis.visit_block(body));
}
$(<V as Visitor<$lt>>::Result::output())?
}
ItemKind::DelegationMac(box DelegationMac { qself, prefix, suffixes, body }) => {
try_visit!(vis.visit_qself(qself));
try_visit!(vis.visit_path(prefix$(${ignore($lt)}, id)?));
if let Some(suffixes) = suffixes {
for (ident, rename) in suffixes {
try_visit!(vis.visit_ident(ident));
if let Some(rename) = rename {
try_visit!(vis.visit_ident(rename));
}
}
}
if let Some(body) = body {
try_visit!(vis.visit_block(body));
}
$(<V as Visitor<$lt>>::Result::output())?
}
}
}
}
fn walk_const_item<$($lt,)? V: $Visitor$(<$lt>)?>(vis: &mut V, item: &$($lt)? $($mut)? ConstItem) $(-> <V as Visitor<$lt>>::Result)? {
let ConstItem { defaultness, ident, generics, ty, expr, define_opaque } = item;
try_visit!(visit_defaultness(vis, defaultness));
try_visit!(vis.visit_ident(ident));
try_visit!(vis.visit_generics(generics));
try_visit!(vis.visit_ty(ty));
visit_opt!(vis, visit_expr, expr);
walk_define_opaques(vis, define_opaque)
}
fn walk_foreign_mod<$($lt,)? V: $Visitor$(<$lt>)?>(vis: &mut V, foreign_mod: &$($lt)? $($mut)? ForeignMod) $(-> <V as Visitor<$lt>>::Result)? {
let ForeignMod { extern_span: _, safety, abi: _, items } = foreign_mod;
try_visit!(visit_safety(vis, safety));
$(${ignore($mut)}
items.flat_map_in_place(|item| vis.flat_map_foreign_item(item));
)?
$(
walk_list!(vis, visit_foreign_item, items);
<V as Visitor<$lt>>::Result::output()
)?
}
fn walk_define_opaques<$($lt,)? V: $Visitor$(<$lt>)?>(
visitor: &mut V,
define_opaque: &$($lt)? $($mut)? Option<ThinVec<(NodeId, Path)>>,
) $(-> <V as Visitor<$lt>>::Result)? {
if let Some(define_opaque) = define_opaque {
for (id, path) in define_opaque {
try_visit!(visit_id(visitor, id));
// FIXME(fee1-dead): look into this weird assymetry
try_visit!(visitor.visit_path(path$(${ignore($lt)}, *id)?));
}
}
$(<V as Visitor<$lt>>::Result::output())?
}
};
}
@ -417,163 +681,6 @@ pub fn walk_trait_ref<'a, V: Visitor<'a>>(visitor: &mut V, trait_ref: &'a TraitR
visitor.visit_path(path, *ref_id)
}
impl WalkItemKind for ItemKind {
type Ctxt = ();
fn walk<'a, V: Visitor<'a>>(
&'a self,
span: Span,
id: NodeId,
vis: &'a Visibility,
_ctxt: Self::Ctxt,
visitor: &mut V,
) -> V::Result {
match self {
ItemKind::ExternCrate(_rename, ident) => try_visit!(visitor.visit_ident(ident)),
ItemKind::Use(use_tree) => try_visit!(visitor.visit_use_tree(use_tree, id, false)),
ItemKind::Static(box StaticItem {
ident,
ty,
safety: _,
mutability: _,
expr,
define_opaque,
}) => {
try_visit!(visitor.visit_ident(ident));
try_visit!(visitor.visit_ty(ty));
visit_opt!(visitor, visit_expr, expr);
try_visit!(walk_define_opaques(visitor, define_opaque));
}
ItemKind::Const(box ConstItem {
defaultness: _,
ident,
generics,
ty,
expr,
define_opaque,
}) => {
try_visit!(visitor.visit_ident(ident));
try_visit!(visitor.visit_generics(generics));
try_visit!(visitor.visit_ty(ty));
visit_opt!(visitor, visit_expr, expr);
try_visit!(walk_define_opaques(visitor, define_opaque));
}
ItemKind::Fn(func) => {
let kind = FnKind::Fn(FnCtxt::Free, vis, &*func);
try_visit!(visitor.visit_fn(kind, span, id));
}
ItemKind::Mod(_unsafety, ident, mod_kind) => {
try_visit!(visitor.visit_ident(ident));
match mod_kind {
ModKind::Loaded(items, _inline, _inner_span, _) => {
walk_list!(visitor, visit_item, items);
}
ModKind::Unloaded => {}
}
}
ItemKind::ForeignMod(ForeignMod { extern_span: _, safety: _, abi: _, items }) => {
walk_list!(visitor, visit_foreign_item, items);
}
ItemKind::GlobalAsm(asm) => try_visit!(visitor.visit_inline_asm(asm)),
ItemKind::TyAlias(box TyAlias {
generics,
ident,
bounds,
ty,
defaultness: _,
where_clauses: _,
}) => {
try_visit!(visitor.visit_ident(ident));
try_visit!(visitor.visit_generics(generics));
walk_list!(visitor, visit_param_bound, bounds, BoundKind::Bound);
visit_opt!(visitor, visit_ty, ty);
}
ItemKind::Enum(ident, enum_definition, generics) => {
try_visit!(visitor.visit_ident(ident));
try_visit!(visitor.visit_generics(generics));
try_visit!(visitor.visit_enum_def(enum_definition));
}
ItemKind::Impl(box Impl {
defaultness: _,
safety: _,
generics,
constness: _,
polarity: _,
of_trait,
self_ty,
items,
}) => {
try_visit!(visitor.visit_generics(generics));
visit_opt!(visitor, visit_trait_ref, of_trait);
try_visit!(visitor.visit_ty(self_ty));
walk_list!(
visitor,
visit_assoc_item,
items,
AssocCtxt::Impl { of_trait: of_trait.is_some() }
);
}
ItemKind::Struct(ident, struct_definition, generics)
| ItemKind::Union(ident, struct_definition, generics) => {
try_visit!(visitor.visit_ident(ident));
try_visit!(visitor.visit_generics(generics));
try_visit!(visitor.visit_variant_data(struct_definition));
}
ItemKind::Trait(box Trait {
safety: _,
is_auto: _,
ident,
generics,
bounds,
items,
}) => {
try_visit!(visitor.visit_ident(ident));
try_visit!(visitor.visit_generics(generics));
walk_list!(visitor, visit_param_bound, bounds, BoundKind::SuperTraits);
walk_list!(visitor, visit_assoc_item, items, AssocCtxt::Trait);
}
ItemKind::TraitAlias(ident, generics, bounds) => {
try_visit!(visitor.visit_ident(ident));
try_visit!(visitor.visit_generics(generics));
walk_list!(visitor, visit_param_bound, bounds, BoundKind::Bound);
}
ItemKind::MacCall(mac) => try_visit!(visitor.visit_mac_call(mac)),
ItemKind::MacroDef(ident, ts) => {
try_visit!(visitor.visit_ident(ident));
try_visit!(visitor.visit_mac_def(ts, id))
}
ItemKind::Delegation(box Delegation {
id,
qself,
path,
ident,
rename,
body,
from_glob: _,
}) => {
try_visit!(visitor.visit_qself(qself));
try_visit!(visitor.visit_path(path, *id));
try_visit!(visitor.visit_ident(ident));
visit_opt!(visitor, visit_ident, rename);
visit_opt!(visitor, visit_block, body);
}
ItemKind::DelegationMac(box DelegationMac { qself, prefix, suffixes, body }) => {
try_visit!(visitor.visit_qself(qself));
try_visit!(visitor.visit_path(prefix, id));
if let Some(suffixes) = suffixes {
for (ident, rename) in suffixes {
visitor.visit_ident(ident);
if let Some(rename) = rename {
visitor.visit_ident(rename);
}
}
}
visit_opt!(visitor, visit_block, body);
}
}
V::Result::output()
}
}
pub fn walk_enum_def<'a, V: Visitor<'a>>(
visitor: &mut V,
EnumDef { variants }: &'a EnumDef,
@ -1121,18 +1228,6 @@ pub fn walk_assoc_item<'a, V: Visitor<'a>>(
walk_item_ctxt(visitor, item, ctxt)
}
fn walk_item_ctxt<'a, V: Visitor<'a>, K: WalkItemKind>(
visitor: &mut V,
item: &'a Item<K>,
ctxt: K::Ctxt,
) -> V::Result {
let Item { id, span, vis, attrs, kind, tokens: _ } = item;
walk_list!(visitor, visit_attribute, attrs);
try_visit!(visitor.visit_vis(vis));
try_visit!(kind.walk(*span, *id, vis, ctxt, visitor));
V::Result::output()
}
pub fn walk_struct_def<'a, V: Visitor<'a>>(
visitor: &mut V,
struct_definition: &'a VariantData,
@ -1455,15 +1550,3 @@ pub fn walk_attr_args<'a, V: Visitor<'a>>(visitor: &mut V, args: &'a AttrArgs) -
}
V::Result::output()
}
fn walk_define_opaques<'a, V: Visitor<'a>>(
visitor: &mut V,
define_opaque: &'a Option<ThinVec<(NodeId, Path)>>,
) -> V::Result {
if let Some(define_opaque) = define_opaque {
for (id, path) in define_opaque {
try_visit!(visitor.visit_path(path, *id));
}
}
V::Result::output()
}

22
compiler/rustc_attr_data_structures/src/version.rs
View file

@ -1,4 +1,5 @@
use std::fmt::{self, Display};
use std::sync::OnceLock;
use rustc_macros::{
Decodable, Encodable, HashStable_Generic, PrintAttribute, current_rustc_version,
@ -16,8 +17,29 @@ pub struct RustcVersion {
impl RustcVersion {
pub const CURRENT: Self = current_rustc_version!();
pub fn current_overridable() -> Self {
*CURRENT_OVERRIDABLE.get_or_init(|| {
if let Ok(override_var) = std::env::var("RUSTC_OVERRIDE_VERSION_STRING")
&& let Some(override_) = Self::parse_str(&override_var)
{
override_
} else {
Self::CURRENT
}
})
}
fn parse_str(value: &str) -> Option<Self> {
// Ignore any suffixes such as "-dev" or "-nightly".
let mut components = value.split('-').next().unwrap().splitn(3, '.');
let major = components.next()?.parse().ok()?;
let minor = components.next()?.parse().ok()?;
let patch = components.next().unwrap_or("0").parse().ok()?;
Some(RustcVersion { major, minor, patch })
}
}
static CURRENT_OVERRIDABLE: OnceLock<RustcVersion> = OnceLock::new();
impl Display for RustcVersion {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(formatter, "{}.{}.{}", self.major, self.minor, self.patch)

4
compiler/rustc_attr_parsing/src/attributes/allow_unstable.rs
View file

@ -10,7 +10,7 @@ use crate::session_diagnostics;
pub(crate) struct AllowInternalUnstableParser;
impl CombineAttributeParser for AllowInternalUnstableParser {
const PATH: &'static [rustc_span::Symbol] = &[sym::allow_internal_unstable];
const PATH: &'static [Symbol] = &[sym::allow_internal_unstable];
type Item = (Symbol, Span);
const CONVERT: ConvertFn<Self::Item> = AttributeKind::AllowInternalUnstable;
@ -24,7 +24,7 @@ impl CombineAttributeParser for AllowInternalUnstableParser {
pub(crate) struct AllowConstFnUnstableParser;
impl CombineAttributeParser for AllowConstFnUnstableParser {
const PATH: &'static [rustc_span::Symbol] = &[sym::rustc_allow_const_fn_unstable];
const PATH: &'static [Symbol] = &[sym::rustc_allow_const_fn_unstable];
type Item = Symbol;
const CONVERT: ConvertFn<Self::Item> = AttributeKind::AllowConstFnUnstable;

4
compiler/rustc_attr_parsing/src/attributes/cfg.rs
View file

@ -129,9 +129,9 @@ pub fn eval_condition(
// See https://github.com/rust-lang/rust/issues/64796#issuecomment-640851454 for details
if sess.psess.assume_incomplete_release {
RustcVersion::CURRENT > min_version
RustcVersion::current_overridable() > min_version
} else {
RustcVersion::CURRENT >= min_version
RustcVersion::current_overridable() >= min_version
}
}
MetaItemKind::List(mis) => {

4
compiler/rustc_attr_parsing/src/attributes/deprecation.rs
View file

@ -42,9 +42,9 @@ fn get(
}
impl SingleAttributeParser for DeprecationParser {
const PATH: &'static [rustc_span::Symbol] = &[sym::deprecated];
const PATH: &'static [Symbol] = &[sym::deprecated];
fn on_duplicate(cx: &AcceptContext<'_>, first_span: rustc_span::Span) {
fn on_duplicate(cx: &AcceptContext<'_>, first_span: Span) {
// FIXME(jdonszelmann): merge with errors from check_attrs.rs
cx.emit_err(session_diagnostics::UnusedMultiple {
this: cx.attr_span,

8
compiler/rustc_attr_parsing/src/attributes/mod.rs
View file

@ -17,7 +17,7 @@
use std::marker::PhantomData;
use rustc_attr_data_structures::AttributeKind;
use rustc_span::Span;
use rustc_span::{Span, Symbol};
use thin_vec::ThinVec;
use crate::context::{AcceptContext, FinalizeContext};
@ -33,7 +33,7 @@ pub(crate) mod transparency;
pub(crate) mod util;
type AcceptFn<T> = fn(&mut T, &AcceptContext<'_>, &ArgParser<'_>);
type AcceptMapping<T> = &'static [(&'static [rustc_span::Symbol], AcceptFn<T>)];
type AcceptMapping<T> = &'static [(&'static [Symbol], AcceptFn<T>)];
/// An [`AttributeParser`] is a type which searches for syntactic attributes.
///
@ -72,7 +72,7 @@ pub(crate) trait AttributeParser: Default + 'static {
/// [`SingleAttributeParser`] can only convert attributes one-to-one, and cannot combine multiple
/// attributes together like is necessary for `#[stable()]` and `#[unstable()]` for example.
pub(crate) trait SingleAttributeParser: 'static {
const PATH: &'static [rustc_span::Symbol];
const PATH: &'static [Symbol];
/// Caled when a duplicate attribute is found.
///
@ -119,7 +119,7 @@ type ConvertFn<E> = fn(ThinVec<E>) -> AttributeKind;
/// [`CombineAttributeParser`] can only convert a single kind of attribute, and cannot combine multiple
/// attributes together like is necessary for `#[stable()]` and `#[unstable()]` for example.
pub(crate) trait CombineAttributeParser: 'static {
const PATH: &'static [rustc_span::Symbol];
const PATH: &'static [Symbol];
type Item;
const CONVERT: ConvertFn<Self::Item>;

6
compiler/rustc_attr_parsing/src/attributes/repr.rs
View file

@ -1,7 +1,7 @@
use rustc_abi::Align;
use rustc_ast::{IntTy, LitIntType, LitKind, UintTy};
use rustc_attr_data_structures::{AttributeKind, IntType, ReprAttr};
use rustc_span::{Span, Symbol, sym};
use rustc_span::{DUMMY_SP, Span, Symbol, sym};
use super::{CombineAttributeParser, ConvertFn};
use crate::context::AcceptContext;
@ -21,7 +21,7 @@ pub(crate) struct ReprParser;
impl CombineAttributeParser for ReprParser {
type Item = (ReprAttr, Span);
const PATH: &'static [rustc_span::Symbol] = &[sym::repr];
const PATH: &'static [Symbol] = &[sym::repr];
const CONVERT: ConvertFn<Self::Item> = AttributeKind::Repr;
fn extend<'a>(
@ -99,7 +99,7 @@ fn parse_repr(cx: &AcceptContext<'_>, param: &MetaItemParser<'_>) -> Option<Repr
let (name, ident_span) = if let Some(ident) = param.path_without_args().word() {
(Some(ident.name), ident.span)
} else {
(None, rustc_span::DUMMY_SP)
(None, DUMMY_SP)
};
let args = param.args();

2
compiler/rustc_attr_parsing/src/attributes/stability.rs
View file

@ -117,7 +117,7 @@ impl AttributeParser for BodyStabilityParser {
pub(crate) struct ConstStabilityIndirectParser;
// FIXME(jdonszelmann): single word attribute group when we have these
impl SingleAttributeParser for ConstStabilityIndirectParser {
const PATH: &'static [rustc_span::Symbol] = &[sym::rustc_const_stable_indirect];
const PATH: &'static [Symbol] = &[sym::rustc_const_stable_indirect];
// ignore
fn on_duplicate(_cx: &AcceptContext<'_>, _first_span: Span) {}

6
compiler/rustc_attr_parsing/src/attributes/transparency.rs
View file

@ -1,6 +1,6 @@
use rustc_attr_data_structures::AttributeKind;
use rustc_span::hygiene::Transparency;
use rustc_span::sym;
use rustc_span::{Span, Symbol, sym};
use super::{AcceptContext, SingleAttributeParser};
use crate::parser::ArgParser;
@ -11,9 +11,9 @@ pub(crate) struct TransparencyParser;
#[allow(rustc::untranslatable_diagnostic)]
#[allow(rustc::diagnostic_outside_of_impl)]
impl SingleAttributeParser for TransparencyParser {
const PATH: &'static [rustc_span::Symbol] = &[sym::rustc_macro_transparency];
const PATH: &'static [Symbol] = &[sym::rustc_macro_transparency];
fn on_duplicate(cx: &crate::context::AcceptContext<'_>, first_span: rustc_span::Span) {
fn on_duplicate(cx: &crate::context::AcceptContext<'_>, first_span: Span) {
cx.dcx().span_err(vec![first_span, cx.attr_span], "multiple macro transparency attributes");
}

62
compiler/rustc_attr_parsing/src/context.rs
View file

@ -26,12 +26,16 @@ macro_rules! attribute_groups {
(
pub(crate) static $name: ident = [$($names: ty),* $(,)?];
) => {
pub(crate) static $name: LazyLock<(
BTreeMap<&'static [Symbol], Vec<Box<dyn Fn(&AcceptContext<'_>, &ArgParser<'_>) + Send + Sync>>>,
Vec<Box<dyn Send + Sync + Fn(&FinalizeContext<'_>) -> Option<AttributeKind>>>
)> = LazyLock::new(|| {
let mut accepts = BTreeMap::<_, Vec<Box<dyn Fn(&AcceptContext<'_>, &ArgParser<'_>) + Send + Sync>>>::new();
let mut finalizes = Vec::<Box<dyn Send + Sync + Fn(&FinalizeContext<'_>) -> Option<AttributeKind>>>::new();
type Accepts = BTreeMap<
&'static [Symbol],
Box<dyn Send + Sync + Fn(&AcceptContext<'_>, &ArgParser<'_>)>
>;
type Finalizes = Vec<
Box<dyn Send + Sync + Fn(&FinalizeContext<'_>) -> Option<AttributeKind>>
>;
pub(crate) static $name: LazyLock<(Accepts, Finalizes)> = LazyLock::new(|| {
let mut accepts = Accepts::new();
let mut finalizes = Finalizes::new();
$(
{
thread_local! {
@ -39,11 +43,12 @@ macro_rules! attribute_groups {
};
for (k, v) in <$names>::ATTRIBUTES {
accepts.entry(*k).or_default().push(Box::new(|cx, args| {
let old = accepts.insert(*k, Box::new(|cx, args| {
STATE_OBJECT.with_borrow_mut(|s| {
v(s, cx, args)
})
}));
assert!(old.is_none());
}
finalizes.push(Box::new(|cx| {
@ -110,7 +115,8 @@ impl<'a> Deref for AcceptContext<'a> {
/// Context given to every attribute parser during finalization.
///
/// Gives [`AttributeParser`](crate::attributes::AttributeParser)s enough information to create errors, for example.
/// Gives [`AttributeParser`](crate::attributes::AttributeParser)s enough information to create
/// errors, for example.
pub(crate) struct FinalizeContext<'a> {
/// The parse context, gives access to the session and the
/// diagnostics context.
@ -141,10 +147,9 @@ pub struct AttributeParser<'sess> {
sess: &'sess Session,
features: Option<&'sess Features>,
/// *only* parse attributes with this symbol.
/// *Only* parse attributes with this symbol.
///
/// Used in cases where we want the lowering infrastructure for
/// parse just a single attribute.
/// Used in cases where we want the lowering infrastructure for parse just a single attribute.
parse_only: Option<Symbol>,
/// Can be used to instruct parsers to reduce the number of diagnostics it emits.
@ -157,9 +162,9 @@ impl<'sess> AttributeParser<'sess> {
/// One example where this is necessary, is to parse `feature` attributes themselves for
/// example.
///
/// Try to use this as little as possible. Attributes *should* be lowered during `rustc_ast_lowering`.
/// Some attributes require access to features to parse, which would crash if you tried to do so
/// through [`parse_limited`](Self::parse_limited).
/// Try to use this as little as possible. Attributes *should* be lowered during
/// `rustc_ast_lowering`. Some attributes require access to features to parse, which would
/// crash if you tried to do so through [`parse_limited`](Self::parse_limited).
///
/// To make sure use is limited, supply a `Symbol` you'd like to parse. Only attributes with
/// that symbol are picked out of the list of instructions and parsed. Those are returned.
@ -217,19 +222,18 @@ impl<'sess> AttributeParser<'sess> {
let group_cx = FinalizeContext { cx: self, target_span };
for attr in attrs {
// if we're only looking for a single attribute,
// skip all the ones we don't care about
// If we're only looking for a single attribute, skip all the ones we don't care about.
if let Some(expected) = self.parse_only {
if !attr.has_name(expected) {
continue;
}
}
// sometimes, for example for `#![doc = include_str!("readme.md")]`,
// Sometimes, for example for `#![doc = include_str!("readme.md")]`,
// doc still contains a non-literal. You might say, when we're lowering attributes
// that's expanded right? But no, sometimes, when parsing attributes on macros,
// we already use the lowering logic and these are still there. So, when `omit_doc`
// is set we *also* want to ignore these
// is set we *also* want to ignore these.
if omit_doc == OmitDoc::Skip && attr.has_name(sym::doc) {
continue;
}
@ -263,21 +267,17 @@ impl<'sess> AttributeParser<'sess> {
let (path, args) = parser.deconstruct();
let parts = path.segments().map(|i| i.name).collect::<Vec<_>>();
if let Some(accepts) = ATTRIBUTE_MAPPING.0.get(parts.as_slice()) {
for f in accepts {
let cx = AcceptContext {
group_cx: &group_cx,
attr_span: lower_span(attr.span),
};
if let Some(accept) = ATTRIBUTE_MAPPING.0.get(parts.as_slice()) {
let cx =
AcceptContext { group_cx: &group_cx, attr_span: lower_span(attr.span) };
f(&cx, &args)
}
accept(&cx, &args)
} else {
// if we're here, we must be compiling a tool attribute... Or someone forgot to
// parse their fancy new attribute. Let's warn them in any case. If you are that
// person, and you really your attribute should remain unparsed, carefully read the
// documentation in this module and if you still think so you can add an exception
// to this assertion.
// If we're here, we must be compiling a tool attribute... Or someone
// forgot to parse their fancy new attribute. Let's warn them in any case.
// If you are that person, and you really think your attribute should
// remain unparsed, carefully read the documentation in this module and if
// you still think so you can add an exception to this assertion.
// FIXME(jdonszelmann): convert other attributes, and check with this that
// we caught em all

3
compiler/rustc_attr_parsing/src/parser.rs
View file

@ -12,8 +12,7 @@ use rustc_ast::{AttrArgs, DelimArgs, Expr, ExprKind, LitKind, MetaItemLit, Norma
use rustc_ast_pretty::pprust;
use rustc_errors::DiagCtxtHandle;
use rustc_hir::{self as hir, AttrPath};
use rustc_span::symbol::{Ident, kw, sym};
use rustc_span::{ErrorGuaranteed, Span, Symbol};
use rustc_span::{ErrorGuaranteed, Ident, Span, Symbol, kw, sym};
pub struct SegmentIterator<'a> {
offset: usize,

6
compiler/rustc_codegen_cranelift/.github/workflows/main.yml vendored
View file

@ -34,9 +34,9 @@ jobs:
- name: Rustfmt
run: |
cargo fmt --check
rustfmt --check build_system/main.rs
rustfmt --check example/*
rustfmt --check scripts/*.rs
rustfmt --check --edition 2024 build_system/main.rs
rustfmt --check --edition 2024 example/*
rustfmt --check --edition 2024 scripts/*.rs
test:

80
compiler/rustc_codegen_cranelift/Cargo.lock
View file

@ -43,39 +43,42 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "cranelift-assembler-x64"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e4b56ebe316895d3fa37775d0a87b0c889cc933f5c8b253dbcc7c7bcb7fe7e4"
checksum = "9ff8e35182c7372df00447cb90a04e584e032c42b9b9b6e8c50ddaaf0d7900d5"
dependencies = [
"cranelift-assembler-x64-meta",
]
[[package]]
name = "cranelift-assembler-x64-meta"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95cabbc01dfbd7dcd6c329ca44f0212910309c221797ac736a67a5bc8857fe1b"
checksum = "14220f9c2698015c3b94dc6b84ae045c1c45509ddc406e43c6139252757fdb7a"
dependencies = [
"cranelift-srcgen",
]
[[package]]
name = "cranelift-bforest"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76ffe46df300a45f1dc6f609dc808ce963f0e3a2e971682c479a2d13e3b9b8ef"
checksum = "d372ef2777ceefd75829e1390211ac240e9196bc60699218f7ea2419038288ee"
dependencies = [
"cranelift-entity",
]
[[package]]
name = "cranelift-bitset"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b265bed7c51e1921fdae6419791d31af77d33662ee56d7b0fa0704dc8d231cab"
checksum = "56323783e423818fa89ce8078e90a3913d2a6e0810399bfce8ebd7ee87baa81f"
[[package]]
name = "cranelift-codegen"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e606230a7e3a6897d603761baee0d19f88d077f17b996bb5089488a29ae96e41"
checksum = "74ffb780aab6186c6e9ba26519654b1ac55a09c0a866f6088a4efbbd84da68ed"
dependencies = [
"bumpalo",
"cranelift-assembler-x64",
@ -98,43 +101,44 @@ dependencies = [
[[package]]
name = "cranelift-codegen-meta"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a63bffafc23bc60969ad528e138788495999d935f0adcfd6543cb151ca8637d"
checksum = "c23ef13814d3b39c869650d5961128cbbecad83fbdff4e6836a03ecf6862d7ed"
dependencies = [
"cranelift-assembler-x64",
"cranelift-assembler-x64-meta",
"cranelift-codegen-shared",
"cranelift-srcgen",
]
[[package]]
name = "cranelift-codegen-shared"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af50281b67324b58e843170a6a5943cf6d387c06f7eeacc9f5696e4ab7ae7d7e"
checksum = "b9f623300657679f847803ce80811454bfff89cea4f6bf684be5c468d4a73631"
[[package]]
name = "cranelift-control"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c20c1b38d1abfbcebb0032e497e71156c0e3b8dcb3f0a92b9863b7bcaec290c"
checksum = "31f4168af69989aa6b91fab46799ed4df6096f3209f4a6c8fb4358f49c60188f"
dependencies = [
"arbitrary",
]
[[package]]
name = "cranelift-entity"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c2c67d95507c51b4a1ff3f3555fe4bfec36b9e13c1b684ccc602736f5d5f4a2"
checksum = "ca6fa9bae1c8de26d71ac2162f069447610fd91e7780cb480ee0d76ac81eabb8"
dependencies = [
"cranelift-bitset",
]
[[package]]
name = "cranelift-frontend"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e002691cc69c38b54fc7ec93e5be5b744f627d027031d991cc845d1d512d0ce"
checksum = "b8219205608aa0b0e6769b580284a7e055c7e0c323c1041cde7ca078add3e412"
dependencies = [
"cranelift-codegen",
"log",
@ -144,15 +148,15 @@ dependencies = [
[[package]]
name = "cranelift-isle"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e93588ed1796cbcb0e2ad160403509e2c5d330d80dd6e0014ac6774c7ebac496"
checksum = "588d0c5964f10860b04043e55aab26d7f7a206b0fd4f10c5260e8aa5773832bd"
[[package]]
name = "cranelift-jit"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17f6682f0b193d6b7873cc8e7ed67e8776a8a26f50eeabf88534e9be618b9a03"
checksum = "56bd917ddc524f84f4066f954062875bdfc0dffea068ee94e906d98de5ac7c33"
dependencies = [
"anyhow",
"cranelift-codegen",
@ -170,9 +174,9 @@ dependencies = [
[[package]]
name = "cranelift-module"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff19784c6de05116e63e6a34791012bd927b2a4eac56233039c46f1b6a4edac8"
checksum = "68a03c057d8a992e06596c871341e446af43ff9224f941e5b8adea39137a5391"
dependencies = [
"anyhow",
"cranelift-codegen",
@ -181,9 +185,9 @@ dependencies = [
[[package]]
name = "cranelift-native"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e5b09bdd6407bf5d89661b80cf926ce731c9e8cc184bf49102267a2369a8358e"
checksum = "19ed3c94cb97b14f92b6a94a1d45ef8c851f6a2ad9114e5d91d233f7da638fed"
dependencies = [
"cranelift-codegen",
"libc",
@ -192,9 +196,9 @@ dependencies = [
[[package]]
name = "cranelift-object"
version = "0.118.0"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "685e8661a30d1cb69509f589ac643adeee79c5f63c0da316431b9fad29e6d3b4"
checksum = "a64dacef362a69375a604f6636e5e9a174fb96dba3b273646fcd9fa85c1d0997"
dependencies = [
"anyhow",
"cranelift-codegen",
@ -205,6 +209,12 @@ dependencies = [
"target-lexicon",
]
[[package]]
name = "cranelift-srcgen"
version = "0.120.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85256fac1519a7d25a040c1d850fba67478f3f021ad5fdf738ba4425ee862dbf"
[[package]]
name = "crc32fast"
version = "1.4.2"
@ -331,9 +341,9 @@ dependencies = [
[[package]]
name = "regalloc2"
version = "0.11.1"
version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "145c1c267e14f20fb0f88aa76a1c5ffec42d592c1d28b3cd9148ae35916158d3"
checksum = "5216b1837de2149f8bc8e6d5f88a9326b63b8c836ed58ce4a0a29ec736a59734"
dependencies = [
"allocator-api2",
"bumpalo",
@ -436,9 +446,9 @@ checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83"
[[package]]
name = "wasmtime-jit-icache-coherence"
version = "31.0.0"
version = "33.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a54f6c6c7e9d7eeee32dfcc10db7f29d505ee7dd28d00593ea241d5f70698e64"
checksum = "175e924dbc944c185808466d1e90b5a7feb610f3b9abdfe26f8ee25fd1086d1c"
dependencies = [
"anyhow",
"cfg-if",

24
compiler/rustc_codegen_cranelift/Cargo.toml
View file

@ -8,12 +8,12 @@ crate-type = ["dylib"]
[dependencies]
# These have to be in sync with each other
cranelift-codegen = { version = "0.118.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] }
cranelift-frontend = { version = "0.118.0" }
cranelift-module = { version = "0.118.0" }
cranelift-native = { version = "0.118.0" }
cranelift-jit = { version = "0.118.0", optional = true }
cranelift-object = { version = "0.118.0" }
cranelift-codegen = { version = "0.120.0", default-features = false, features = ["std", "timing", "unwind", "all-native-arch"] }
cranelift-frontend = { version = "0.120.0" }
cranelift-module = { version = "0.120.0" }
cranelift-native = { version = "0.120.0" }
cranelift-jit = { version = "0.120.0", optional = true }
cranelift-object = { version = "0.120.0" }
target-lexicon = "0.13"
gimli = { version = "0.31", default-features = false, features = ["write"] }
object = { version = "0.36", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] }
@ -24,12 +24,12 @@ smallvec = "1.8.1"
[patch.crates-io]
# Uncomment to use an unreleased version of cranelift
#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" }
#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" }
#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" }
#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" }
#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" }
#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-31.0.0", version = "0.118.0" }
#cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
#cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
#cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
#cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
#cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
#cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime.git", branch = "release-33.0.0", version = "0.120.0" }
# Uncomment to use local checkout of cranelift
#cranelift-codegen = { path = "../wasmtime/cranelift/codegen" }

4
compiler/rustc_codegen_cranelift/Readme.md
View file

@ -7,7 +7,7 @@ If not please open an issue.
## Download using Rustup
The Cranelift codegen backend is distributed in nightly builds on Linux and x86_64 macOS. If you want to
The Cranelift codegen backend is distributed in nightly builds on Linux, macOS and x86_64 Windows. If you want to
install it using Rustup, you can do that by running:
```bash
@ -79,7 +79,7 @@ For more docs on how to build and test see [build_system/usage.txt](build_system
Not all targets are available as rustup component for nightly. See notes in the platform support matrix.
[^xcoff]: XCOFF object file format is not supported.
[^no-rustup]: Not available as rustup component for nightly. You can build it yourself.
[^no-rustup]: Not available as [rustup component for nightly](https://rust-lang.github.io/rustup-components-history/). You can build it yourself.
## Usage

2
compiler/rustc_codegen_cranelift/build_system/build_backend.rs
View file

@ -18,7 +18,7 @@ pub(crate) fn build_backend(
let mut cmd = CG_CLIF.build(&bootstrap_host_compiler, dirs);
let mut rustflags = rustflags_from_env("RUSTFLAGS");
rustflags.push("-Zallow-features=rustc_private".to_owned());
rustflags.push("-Zallow-features=rustc_private,f16,f128".to_owned());
rustflags_to_cmd_env(&mut cmd, "RUSTFLAGS", &rustflags);
if env::var("CG_CLIF_EXPENSIVE_CHECKS").is_ok() {

5
compiler/rustc_codegen_cranelift/build_system/build_sysroot.rs
View file

@ -168,7 +168,8 @@ fn build_llvm_sysroot_for_triple(compiler: Compiler) -> SysrootTarget {
let file_name_str = file.file_name().unwrap().to_str().unwrap();
if (file_name_str.contains("rustc_")
&& !file_name_str.contains("rustc_std_workspace_")
&& !file_name_str.contains("rustc_demangle"))
&& !file_name_str.contains("rustc_demangle")
&& !file_name_str.contains("rustc_literal_escaper"))
|| file_name_str.contains("chalk")
|| file_name_str.contains("tracing")
|| file_name_str.contains("regex")
@ -234,7 +235,7 @@ fn build_clif_sysroot_for_triple(
compiler.rustflags.extend(rustflags);
let mut build_cmd = STANDARD_LIBRARY.build(&compiler, dirs);
build_cmd.arg("--release");
build_cmd.arg("--features").arg("backtrace panic-unwind compiler-builtins-no-f16-f128");
build_cmd.arg("--features").arg("backtrace panic-unwind");
build_cmd.arg(format!("-Zroot-dir={}", STDLIB_SRC.to_path(dirs).display()));
build_cmd.env("CARGO_PROFILE_RELEASE_DEBUG", "true");
build_cmd.env("__CARGO_DEFAULT_LIB_METADATA", "cg_clif");

2
compiler/rustc_codegen_cranelift/docs/usage.md
View file

@ -47,7 +47,7 @@ These are a few functions that allow you to easily run rust code from the shell
```bash
function jit_naked() {
echo "$@" | $cg_clif_dir/dist/rustc-clif - -Zunstable-options -Cllvm-args=jit-mode-Cprefer-dynamic
echo "$@" | $cg_clif_dir/dist/rustc-clif - -Zunstable-options -Cllvm-args=jit-mode -Cprefer-dynamic
}
function jit() {

20
compiler/rustc_codegen_cranelift/example/mini_core.rs
View file

@ -521,10 +521,28 @@ fn panic_cannot_unwind() -> ! {
}
#[lang = "eh_personality"]
fn eh_personality() -> ! {
// FIXME personality signature depends on target
fn eh_personality(
_version: i32,
_actions: i32,
_exception_class: u64,
_exception_object: *mut (),
_context: *mut (),
) -> i32 {
loop {}
}
#[lang = "panic_in_cleanup"]
fn panic_in_cleanup() -> ! {
loop {}
}
#[cfg(all(unix, not(target_vendor = "apple")))]
#[link(name = "gcc_s")]
extern "C" {
fn _Unwind_Resume(exc: *mut ()) -> !;
}
#[lang = "drop_in_place"]
#[allow(unconditional_recursion)]
pub unsafe fn drop_in_place<T: ?Sized>(to_drop: *mut T) {

3
compiler/rustc_codegen_cranelift/example/std_example.rs
View file

@ -166,6 +166,7 @@ fn main() {
enum Never {}
}
#[cfg(not(target_arch = "s390x"))] // s390x doesn't have vector instructions enabled by default
foo(I64X2([0, 0]));
transmute_wide_pointer();
@ -203,9 +204,11 @@ fn rust_call_abi() {
rust_call_abi_callee((1, 2));
}
#[cfg_attr(target_arch = "s390x", allow(dead_code))]
#[repr(simd)]
struct I64X2([i64; 2]);
#[cfg_attr(target_arch = "s390x", allow(dead_code))]
#[allow(improper_ctypes_definitions)]
extern "C" fn foo(_a: I64X2) {}

9
compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch
View file

@ -37,6 +37,15 @@ diff --git a/library/core/src/sync/atomic.rs b/library/core/src/sync/atomic.rs
index bf2b6d59f88..d5ccce03bbf 100644
--- a/library/core/src/sync/atomic.rs
+++ b/library/core/src/sync/atomic.rs
@@ -300,8 +300,6 @@ impl_atomic_primitive!(AtomicI32(i32), size("32"), align(4));
impl_atomic_primitive!(AtomicU32(u32), size("32"), align(4));
impl_atomic_primitive!(AtomicI64(i64), size("64"), align(8));
impl_atomic_primitive!(AtomicU64(u64), size("64"), align(8));
-impl_atomic_primitive!(AtomicI128(i128), size("128"), align(16));
-impl_atomic_primitive!(AtomicU128(u128), size("128"), align(16));
#[cfg(target_pointer_width = "16")]
impl_atomic_primitive!(AtomicIsize(isize), size("ptr"), align(2));
@@ -3585,44 +3585,6 @@ pub const fn as_ptr(&self) -> *mut $int_type {
8,
u64 AtomicU64

2
compiler/rustc_codegen_cranelift/rust-toolchain
View file

@ -1,4 +1,4 @@
[toolchain]
channel = "nightly-2025-03-30"
channel = "nightly-2025-05-25"
components = ["rust-src", "rustc-dev", "llvm-tools"]
profile = "minimal"

2
compiler/rustc_codegen_cranelift/scripts/setup_rust_fork.sh
View file

@ -43,7 +43,7 @@ verbose-tests = false
# disabled bootstrap will crash trying to copy llvm tools for the bootstrap
# compiler.
llvm-tools = false
std-features = ["panic-unwind", "compiler-builtins-no-f16-f128"]
std-features = ["panic-unwind"]
EOF

17
compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh
View file

@ -53,6 +53,8 @@ rm -r tests/run-make/split-debuginfo # same
rm -r tests/run-make/target-specs # i686 not supported by Cranelift
rm -r tests/run-make/mismatching-target-triples # same
rm tests/ui/asm/x86_64/issue-96797.rs # const and sym inline asm operands don't work entirely correctly
rm tests/ui/asm/global-asm-mono-sym-fn.rs # same
rm tests/ui/asm/naked-asm-mono-sym-fn.rs # same
rm tests/ui/asm/x86_64/goto.rs # inline asm labels not supported
rm tests/ui/simd/simd-bitmask-notpow2.rs # non-pow-of-2 simd vector sizes
rm -r tests/run-make/embed-source-dwarf # embedding sources in debuginfo
@ -70,19 +72,13 @@ rm tests/ui/consts/precise-drop-with-coverage.rs
rm tests/ui/issues/issue-85461.rs
rm -r tests/ui/instrument-coverage/
# missing f16/f128 support
rm tests/ui/half-open-range-patterns/half-open-range-pats-semantics.rs
rm tests/ui/asm/aarch64/type-f16.rs
rm tests/ui/float/conv-bits-runtime-const.rs
rm tests/ui/consts/const-eval/float_methods.rs
rm tests/ui/match/match-float.rs
# optimization tests
# ==================
rm tests/ui/codegen/issue-28950.rs # depends on stack size optimizations
rm tests/ui/codegen/init-large-type.rs # same
rm -r tests/run-make/fmt-write-bloat/ # tests an optimization
rm tests/ui/statics/const_generics.rs # same
rm tests/ui/linking/executable-no-mangle-strip.rs # requires --gc-sections to work for statics
# backend specific tests
# ======================
@ -96,6 +92,7 @@ rm -r tests/run-make/llvm-location-discriminator-limit-dummy-span # same
rm tests/ui/abi/stack-protector.rs # requires stack protector support
rm -r tests/run-make/emit-stack-sizes # requires support for -Z emit-stack-sizes
rm -r tests/run-make/optimization-remarks-dir # remarks are LLVM specific
rm -r tests/ui/optimization-remark.rs # same
rm -r tests/run-make/print-to-output # requires --print relocation-models
# requires asm, llvm-ir and/or llvm-bc emit support
@ -118,7 +115,6 @@ rm tests/ui/mir/mir_raw_fat_ptr.rs # same
rm tests/ui/consts/issue-33537.rs # same
rm tests/ui/consts/const-mut-refs-crate.rs # same
rm tests/ui/abi/large-byval-align.rs # exceeds implementation limit of Cranelift
rm tests/ui/abi/simd-abi-checks-avx.rs # attempts to declare function with two different signatures
# doesn't work due to the way the rustc test suite is invoked.
# should work when using ./x.py test the way it is intended
@ -129,7 +125,11 @@ rm -r tests/run-make/compiler-builtins # Expects lib/rustlib/src/rust to contain
rm -r tests/run-make/translation # same
rm -r tests/run-make/missing-unstable-trait-bound # This disables support for unstable features, but running cg_clif needs some unstable features
rm -r tests/run-make/const-trait-stable-toolchain # same
rm -r tests/run-make/print-request-help-stable-unstable # same
rm -r tests/run-make/incr-add-rust-src-component
rm tests/ui/errors/remap-path-prefix-sysroot.rs # different sysroot source path
rm -r tests/run-make/export/extern-opt # something about rustc version mismatches
rm -r tests/run-make/export # same
# genuine bugs
# ============
@ -141,6 +141,7 @@ rm -r tests/run-make/panic-abort-eh_frame # .eh_frame emitted with panic=abort
rm tests/ui/process/nofile-limit.rs # TODO some AArch64 linking issue
rm tests/ui/backtrace/synchronized-panic-handler.rs # missing needs-unwind annotation
rm tests/ui/lint/non-snake-case/lint-non-snake-case-crate.rs # same
rm tests/ui/async-await/async-drop/async-drop-initial.rs # same (rust-lang/rust#140493)
rm -r tests/ui/codegen/equal-pointers-unequal # make incorrect assumptions about the location of stack variables
rm tests/ui/stdio-is-blocking.rs # really slow with unoptimized libstd

68
compiler/rustc_codegen_cranelift/src/abi/mod.rs
View file

@ -20,6 +20,7 @@ use rustc_middle::ty::print::with_no_trimmed_paths;
use rustc_session::Session;
use rustc_span::source_map::Spanned;
use rustc_target::callconv::{Conv, FnAbi, PassMode};
use smallvec::SmallVec;
use self::pass_mode::*;
pub(crate) use self::returning::codegen_return;
@ -153,10 +154,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
let ret = self.lib_call_unadjusted(name, params, returns, &args)[0];
// FIXME(bytecodealliance/wasmtime#6104) use bitcast instead of store to get from i64x2 to i128
let ret_ptr = self.create_stack_slot(16, 16);
ret_ptr.store(self, ret, MemFlags::trusted());
Cow::Owned(vec![ret_ptr.load(self, types::I128, MemFlags::trusted())])
Cow::Owned(vec![codegen_bitcast(self, types::I128, ret)])
} else if ret_single_i128 && self.tcx.sess.target.arch == "s390x" {
// Return i128 using a return area pointer on s390x.
let mut params = params;
@ -184,11 +182,9 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
let sig = Signature { params, returns, call_conv: self.target_config.default_call_conv };
let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap();
let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func);
if self.clif_comments.enabled() {
self.add_comment(func_ref, format!("{:?}", name));
}
let call_inst = self.bcx.ins().call(func_ref, args);
if self.clif_comments.enabled() {
self.add_comment(func_ref, format!("{:?}", name));
self.add_comment(call_inst, format!("lib_call {}", name));
}
let results = self.bcx.inst_results(call_inst);
@ -384,6 +380,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
args: &[Spanned<Operand<'tcx>>],
destination: Place<'tcx>,
target: Option<BasicBlock>,
_unwind: UnwindAction,
) {
let func = codegen_operand(fx, func);
let fn_sig = func.layout().ty.fn_sig(fx.tcx);
@ -529,7 +526,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
Some(Instance { def: InstanceKind::Virtual(_, idx), .. }) => {
if fx.clif_comments.enabled() {
let nop_inst = fx.bcx.ins().nop();
fx.add_comment(
fx.add_post_comment(
nop_inst,
with_no_trimmed_paths!(format!(
"virtual call; self arg pass mode: {:?}",
@ -555,7 +552,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
None => {
if fx.clif_comments.enabled() {
let nop_inst = fx.bcx.ins().nop();
fx.add_comment(nop_inst, "indirect call");
fx.add_post_comment(nop_inst, "indirect call");
}
let func = func.load_scalar(fx);
@ -585,17 +582,18 @@ pub(crate) fn codegen_terminator_call<'tcx>(
adjust_call_for_c_variadic(fx, &fn_abi, source_info, func_ref, &mut call_args);
}
if fx.clif_comments.enabled() {
let nop_inst = fx.bcx.ins().nop();
with_no_trimmed_paths!(fx.add_comment(nop_inst, format!("abi: {:?}", fn_abi)));
}
match func_ref {
let call_inst = match func_ref {
CallTarget::Direct(func_ref) => fx.bcx.ins().call(func_ref, &call_args),
CallTarget::Indirect(sig, func_ptr) => {
fx.bcx.ins().call_indirect(sig, func_ptr, &call_args)
}
};
if fx.clif_comments.enabled() {
with_no_trimmed_paths!(fx.add_comment(call_inst, format!("abi: {:?}", fn_abi)));
}
fx.bcx.func.dfg.inst_results(call_inst).iter().copied().collect::<SmallVec<[Value; 2]>>()
});
if let Some(dest) = target {
@ -705,13 +703,16 @@ pub(crate) fn codegen_drop<'tcx>(
source_info: mir::SourceInfo,
drop_place: CPlace<'tcx>,
target: BasicBlock,
_unwind: UnwindAction,
) {
let ty = drop_place.layout().ty;
let drop_instance = Instance::resolve_drop_in_place(fx.tcx, ty);
let ret_block = fx.get_block(target);
// AsyncDropGlueCtorShim can't be here
if let ty::InstanceKind::DropGlue(_, None) = drop_instance.def {
// we don't actually need to drop anything
fx.bcx.ins().jump(ret_block, &[]);
} else {
match ty.kind() {
ty::Dynamic(_, _, ty::Dyn) => {
@ -748,7 +749,9 @@ pub(crate) fn codegen_drop<'tcx>(
let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
let sig = fx.bcx.import_signature(sig);
// FIXME implement cleanup on exceptions
fx.bcx.ins().call_indirect(sig, drop_fn, &[ptr]);
fx.bcx.ins().jump(ret_block, &[]);
}
ty::Dynamic(_, _, ty::DynStar) => {
// IN THIS ARM, WE HAVE:
@ -792,6 +795,8 @@ pub(crate) fn codegen_drop<'tcx>(
let sig = clif_sig_from_fn_abi(fx.tcx, fx.target_config.default_call_conv, &fn_abi);
let sig = fx.bcx.import_signature(sig);
fx.bcx.ins().call_indirect(sig, drop_fn, &[data]);
// FIXME implement cleanup on exceptions
fx.bcx.ins().jump(ret_block, &[]);
}
_ => {
assert!(!matches!(drop_instance.def, InstanceKind::Virtual(_, _)));
@ -817,10 +822,37 @@ pub(crate) fn codegen_drop<'tcx>(
let func_ref = fx.get_function_ref(drop_instance);
fx.bcx.ins().call(func_ref, &call_args);
// FIXME implement cleanup on exceptions
fx.bcx.ins().jump(ret_block, &[]);
}
}
}
let target_block = fx.get_block(target);
fx.bcx.ins().jump(target_block, &[]);
}
pub(crate) fn lib_call_arg_param(tcx: TyCtxt<'_>, ty: Type, is_signed: bool) -> AbiParam {
let param = AbiParam::new(ty);
if ty.is_int() && u64::from(ty.bits()) < tcx.data_layout.pointer_size.bits() {
match (&*tcx.sess.target.arch, &*tcx.sess.target.vendor) {
("x86_64", _) | ("aarch64", "apple") => match (ty, is_signed) {
(types::I8 | types::I16, true) => param.sext(),
(types::I8 | types::I16, false) => param.uext(),
_ => param,
},
("aarch64", _) => param,
("riscv64", _) => match (ty, is_signed) {
(types::I32, _) | (_, true) => param.sext(),
_ => param.uext(),
},
("s390x", _) => {
if is_signed {
param.sext()
} else {
param.uext()
}
}
_ => unimplemented!("{:?}", tcx.sess.target.arch),
}
} else {
param
}
}

2
compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
View file

@ -22,8 +22,10 @@ fn reg_to_abi_param(reg: Reg) -> AbiParam {
(RegKind::Integer, 3..=4) => types::I32,
(RegKind::Integer, 5..=8) => types::I64,
(RegKind::Integer, 9..=16) => types::I128,
(RegKind::Float, 2) => types::F16,
(RegKind::Float, 4) => types::F32,
(RegKind::Float, 8) => types::F64,
(RegKind::Float, 16) => types::F128,
(RegKind::Vector, size) => types::I8.by(u32::try_from(size).unwrap()).unwrap(),
_ => unreachable!("{:?}", reg),
};

12
compiler/rustc_codegen_cranelift/src/abi/returning.rs
View file

@ -46,7 +46,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
ret_arg_abi: &ArgAbi<'tcx, Ty<'tcx>>,
ret_place: CPlace<'tcx>,
f: impl FnOnce(&mut FunctionCx<'_, '_, 'tcx>, Option<Value>) -> Inst,
f: impl FnOnce(&mut FunctionCx<'_, '_, 'tcx>, Option<Value>) -> SmallVec<[Value; 2]>,
) {
let (ret_temp_place, return_ptr) = match ret_arg_abi.mode {
PassMode::Ignore => (None, None),
@ -67,23 +67,21 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast { .. } => (None, None),
};
let call_inst = f(fx, return_ptr);
let results = f(fx, return_ptr);
match ret_arg_abi.mode {
PassMode::Ignore => {}
PassMode::Direct(_) => {
let ret_val = fx.bcx.inst_results(call_inst)[0];
let ret_val = results[0];
ret_place.write_cvalue(fx, CValue::by_val(ret_val, ret_arg_abi.layout));
}
PassMode::Pair(_, _) => {
let ret_val_a = fx.bcx.inst_results(call_inst)[0];
let ret_val_b = fx.bcx.inst_results(call_inst)[1];
let ret_val_a = results[0];
let ret_val_b = results[1];
ret_place
.write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_arg_abi.layout));
}
PassMode::Cast { ref cast, .. } => {
let results =
fx.bcx.inst_results(call_inst).iter().copied().collect::<SmallVec<[Value; 2]>>();
let result =
super::pass_mode::from_casted_value(fx, &results, ret_place.layout(), cast);
ret_place.write_cvalue(fx, result);

69
compiler/rustc_codegen_cranelift/src/base.rs
View file

@ -15,9 +15,9 @@ use rustc_middle::ty::print::with_no_trimmed_paths;
use crate::constant::ConstantCx;
use crate::debuginfo::{FunctionDebugContext, TypeDebugContext};
use crate::enable_verifier;
use crate::prelude::*;
use crate::pretty_clif::CommentWriter;
use crate::{codegen_f16_f128, enable_verifier};
pub(crate) struct CodegenedFunction {
symbol_name: String,
@ -193,6 +193,18 @@ pub(crate) fn compile_fn(
name = codegened_func.symbol_name
));
}
Err(ModuleError::Compilation(CodegenError::Verifier(err))) => {
let early_dcx = rustc_session::EarlyDiagCtxt::new(
rustc_session::config::ErrorOutputType::default(),
);
let _ = early_dcx.early_err(format!("{:?}", err));
let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error(
&context.func,
Some(Box::new(&clif_comments)),
err,
);
early_dcx.early_fatal(format!("cranelift verify error:\n{}", pretty_error));
}
Err(err) => {
panic!("Error while defining {name}: {err:?}", name = codegened_func.symbol_name);
}
@ -303,7 +315,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap();
});
let inst = fx.bcx.func.layout.last_inst(block).unwrap();
fx.add_comment(inst, terminator_head);
fx.add_post_comment(inst, terminator_head);
}
let source_info = bb_data.terminator().source_info;
@ -337,7 +349,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
TerminatorKind::Return => {
crate::abi::codegen_return(fx);
}
TerminatorKind::Assert { cond, expected, msg, target, unwind: _ } => {
TerminatorKind::Assert { cond, expected, msg, target, unwind } => {
if !fx.tcx.sess.overflow_checks() && msg.is_optional_overflow_check() {
let target = fx.get_block(*target);
fx.bcx.ins().jump(target, &[]);
@ -367,6 +379,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
fx,
rustc_hir::LangItem::PanicBoundsCheck,
&[index, len, location],
*unwind,
Some(source_info.span),
);
}
@ -379,6 +392,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
fx,
rustc_hir::LangItem::PanicMisalignedPointerDereference,
&[required, found, location],
*unwind,
Some(source_info.span),
);
}
@ -389,6 +403,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
fx,
rustc_hir::LangItem::PanicNullPointerDereference,
&[location],
*unwind,
Some(source_info.span),
)
}
@ -399,6 +414,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
fx,
msg.panic_function(),
&[location],
*unwind,
Some(source_info.span),
);
}
@ -457,7 +473,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
destination,
target,
fn_span,
unwind: _,
unwind,
call_source: _,
} => {
fx.tcx.prof.generic_activity("codegen call").run(|| {
@ -468,6 +484,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
args,
*destination,
*target,
*unwind,
)
});
}
@ -514,7 +531,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
);
}
TerminatorKind::UnwindTerminate(reason) => {
codegen_unwind_terminate(fx, source_info, *reason);
codegen_unwind_terminate(fx, Some(source_info.span), *reason);
}
TerminatorKind::UnwindResume => {
// FIXME implement unwinding
@ -530,23 +547,19 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
| TerminatorKind::CoroutineDrop => {
bug!("shouldn't exist at codegen {:?}", bb_data.terminator());
}
TerminatorKind::Drop { place, target, unwind: _, replace: _, drop, async_fut } => {
TerminatorKind::Drop { place, target, unwind, replace: _, drop, async_fut } => {
assert!(
async_fut.is_none() && drop.is_none(),
"Async Drop must be expanded or reset to sync before codegen"
);
let drop_place = codegen_place(fx, *place);
crate::abi::codegen_drop(fx, source_info, drop_place, *target);
crate::abi::codegen_drop(fx, source_info, drop_place, *target, *unwind);
}
};
}
}
fn codegen_stmt<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
#[allow(unused_variables)] cur_block: Block,
stmt: &Statement<'tcx>,
) {
fn codegen_stmt<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, cur_block: Block, stmt: &Statement<'tcx>) {
let _print_guard = crate::PrintOnPanic(|| format!("stmt {:?}", stmt));
fx.set_debug_loc(stmt.source_info);
@ -557,7 +570,7 @@ fn codegen_stmt<'tcx>(
if fx.clif_comments.enabled() {
let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap();
with_no_trimmed_paths!({
fx.add_comment(inst, format!("{:?}", stmt));
fx.add_post_comment(inst, format!("{:?}", stmt));
});
}
}
@ -622,6 +635,15 @@ fn codegen_stmt<'tcx>(
let val = operand.load_scalar(fx);
match layout.ty.kind() {
ty::Int(_) => CValue::by_val(fx.bcx.ins().ineg(val), layout),
// FIXME(bytecodealliance/wasmtime#8312): Remove
// once backend lowerings have been added to
// Cranelift.
ty::Float(FloatTy::F16) => {
CValue::by_val(codegen_f16_f128::neg_f16(fx, val), layout)
}
ty::Float(FloatTy::F128) => {
CValue::by_val(codegen_f16_f128::neg_f128(fx, val), layout)
}
ty::Float(_) => CValue::by_val(fx.bcx.ins().fneg(val), layout),
_ => unreachable!("un op Neg for {:?}", layout.ty),
}
@ -793,7 +815,7 @@ fn codegen_stmt<'tcx>(
let done_block = fx.bcx.create_block();
let index = fx.bcx.append_block_param(loop_block, fx.pointer_type);
let zero = fx.bcx.ins().iconst(fx.pointer_type, 0);
fx.bcx.ins().jump(loop_block, &[zero]);
fx.bcx.ins().jump(loop_block, &[zero.into()]);
fx.bcx.switch_to_block(loop_block);
let done = fx.bcx.ins().icmp_imm(IntCC::Equal, index, times as i64);
@ -803,7 +825,7 @@ fn codegen_stmt<'tcx>(
let to = lval.place_index(fx, index);
to.write_cvalue(fx, operand);
let index = fx.bcx.ins().iadd_imm(index, 1);
fx.bcx.ins().jump(loop_block, &[index]);
fx.bcx.ins().jump(loop_block, &[index.into()]);
fx.bcx.switch_to_block(done_block);
fx.bcx.ins().nop();
@ -1058,23 +1080,28 @@ pub(crate) fn codegen_panic_nounwind<'tcx>(
let msg_len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(msg_str.len()).unwrap());
let args = [msg_ptr, msg_len];
codegen_panic_inner(fx, rustc_hir::LangItem::PanicNounwind, &args, span);
codegen_panic_inner(
fx,
rustc_hir::LangItem::PanicNounwind,
&args,
UnwindAction::Terminate(UnwindTerminateReason::Abi),
span,
);
}
pub(crate) fn codegen_unwind_terminate<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
source_info: mir::SourceInfo,
span: Option<Span>,
reason: UnwindTerminateReason,
) {
let args = [];
codegen_panic_inner(fx, reason.lang_item(), &args, Some(source_info.span));
codegen_panic_inner(fx, reason.lang_item(), &[], UnwindAction::Unreachable, span);
}
fn codegen_panic_inner<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
lang_item: rustc_hir::LangItem,
args: &[Value],
_unwind: UnwindAction,
span: Option<Span>,
) {
fx.bcx.set_cold_block(fx.bcx.current_block().unwrap());
@ -1090,6 +1117,8 @@ fn codegen_panic_inner<'tcx>(
let symbol_name = fx.tcx.symbol_name(instance).name;
// FIXME implement cleanup on exceptions
fx.lib_call(
symbol_name,
args.iter().map(|&arg| AbiParam::new(fx.bcx.func.dfg.value_type(arg))).collect(),

21
compiler/rustc_codegen_cranelift/src/cast.rs
View file

@ -1,5 +1,6 @@
//! Various number casting functions
use crate::codegen_f16_f128;
use crate::prelude::*;
pub(crate) fn clif_intcast(
@ -36,6 +37,14 @@ pub(crate) fn clif_int_or_float_cast(
) -> Value {
let from_ty = fx.bcx.func.dfg.value_type(from);
// FIXME(bytecodealliance/wasmtime#8312): Remove in favour of native
// Cranelift operations once Cranelift backends have lowerings for them.
if matches!(from_ty, types::F16 | types::F128)
|| matches!(to_ty, types::F16 | types::F128) && from_ty != to_ty
{
return codegen_f16_f128::codegen_cast(fx, from, from_signed, to_ty, to_signed);
}
if from_ty.is_int() && to_ty.is_int() {
// int-like -> int-like
clif_intcast(
@ -58,8 +67,10 @@ pub(crate) fn clif_int_or_float_cast(
"__float{sign}ti{flt}f",
sign = if from_signed { "" } else { "un" },
flt = match to_ty {
types::F16 => "h",
types::F32 => "s",
types::F64 => "d",
types::F128 => "t",
_ => unreachable!("{:?}", to_ty),
},
);
@ -90,8 +101,10 @@ pub(crate) fn clif_int_or_float_cast(
"__fix{sign}{flt}fti",
sign = if to_signed { "" } else { "uns" },
flt = match from_ty {
types::F16 => "h",
types::F32 => "s",
types::F64 => "d",
types::F128 => "t",
_ => unreachable!("{:?}", to_ty),
},
);
@ -145,8 +158,12 @@ pub(crate) fn clif_int_or_float_cast(
} else if from_ty.is_float() && to_ty.is_float() {
// float -> float
match (from_ty, to_ty) {
(types::F32, types::F64) => fx.bcx.ins().fpromote(types::F64, from),
(types::F64, types::F32) => fx.bcx.ins().fdemote(types::F32, from),
(types::F16, types::F32 | types::F64 | types::F128)
| (types::F32, types::F64 | types::F128)
| (types::F64, types::F128) => fx.bcx.ins().fpromote(to_ty, from),
(types::F128, types::F64 | types::F32 | types::F16)
| (types::F64, types::F32 | types::F16)
| (types::F32, types::F16) => fx.bcx.ins().fdemote(to_ty, from),
_ => from,
}
} else {

284
compiler/rustc_codegen_cranelift/src/codegen_f16_f128.rs Normal file
View file

@ -0,0 +1,284 @@
use crate::prelude::*;
pub(crate) fn f16_to_f32(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
let (value, arg_ty) =
if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" {
(
fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value),
lib_call_arg_param(fx.tcx, types::I16, false),
)
} else {
(value, AbiParam::new(types::F16))
};
fx.lib_call("__extendhfsf2", vec![arg_ty], vec![AbiParam::new(types::F32)], &[value])[0]
}
fn f16_to_f64(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
let ret = f16_to_f32(fx, value);
fx.bcx.ins().fpromote(types::F64, ret)
}
pub(crate) fn f32_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
let ret_ty = if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" {
types::I16
} else {
types::F16
};
let ret = fx.lib_call(
"__truncsfhf2",
vec![AbiParam::new(types::F32)],
vec![AbiParam::new(ret_ty)],
&[value],
)[0];
if ret_ty == types::I16 { fx.bcx.ins().bitcast(types::F16, MemFlags::new(), ret) } else { ret }
}
fn f64_to_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
let ret_ty = if fx.tcx.sess.target.vendor == "apple" && fx.tcx.sess.target.arch == "x86_64" {
types::I16
} else {
types::F16
};
let ret = fx.lib_call(
"__truncdfhf2",
vec![AbiParam::new(types::F64)],
vec![AbiParam::new(ret_ty)],
&[value],
)[0];
if ret_ty == types::I16 { fx.bcx.ins().bitcast(types::F16, MemFlags::new(), ret) } else { ret }
}
pub(crate) fn fcmp(fx: &mut FunctionCx<'_, '_, '_>, cc: FloatCC, lhs: Value, rhs: Value) -> Value {
let ty = fx.bcx.func.dfg.value_type(lhs);
match ty {
types::F32 | types::F64 => fx.bcx.ins().fcmp(cc, lhs, rhs),
types::F16 => {
let lhs = f16_to_f32(fx, lhs);
let rhs = f16_to_f32(fx, rhs);
fx.bcx.ins().fcmp(cc, lhs, rhs)
}
types::F128 => {
let (name, int_cc) = match cc {
FloatCC::Equal => ("__eqtf2", IntCC::Equal),
FloatCC::NotEqual => ("__netf2", IntCC::NotEqual),
FloatCC::LessThan => ("__lttf2", IntCC::SignedLessThan),
FloatCC::LessThanOrEqual => ("__letf2", IntCC::SignedLessThanOrEqual),
FloatCC::GreaterThan => ("__gttf2", IntCC::SignedGreaterThan),
FloatCC::GreaterThanOrEqual => ("__getf2", IntCC::SignedGreaterThanOrEqual),
_ => unreachable!("not currently used in rustc_codegen_cranelift: {cc:?}"),
};
let res = fx.lib_call(
name,
vec![AbiParam::new(types::F128), AbiParam::new(types::F128)],
// FIXME(rust-lang/compiler-builtins#919): This should be `I64` on non-AArch64
// architectures, but switching it before compiler-builtins is fixed causes test
// failures.
vec![AbiParam::new(types::I32)],
&[lhs, rhs],
)[0];
let zero = fx.bcx.ins().iconst(types::I32, 0);
let res = fx.bcx.ins().icmp(int_cc, res, zero);
res
}
_ => unreachable!("{ty:?}"),
}
}
pub(crate) fn codegen_f128_binop(
fx: &mut FunctionCx<'_, '_, '_>,
bin_op: BinOp,
lhs: Value,
rhs: Value,
) -> Value {
let name = match bin_op {
BinOp::Add => "__addtf3",
BinOp::Sub => "__subtf3",
BinOp::Mul => "__multf3",
BinOp::Div => "__divtf3",
_ => unreachable!("handled in `codegen_float_binop`"),
};
fx.lib_call(
name,
vec![AbiParam::new(types::F128), AbiParam::new(types::F128)],
vec![AbiParam::new(types::F128)],
&[lhs, rhs],
)[0]
}
pub(crate) fn neg_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
let bits = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value);
let bits = fx.bcx.ins().bxor_imm(bits, 0x8000);
fx.bcx.ins().bitcast(types::F16, MemFlags::new(), bits)
}
pub(crate) fn neg_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
let bits = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), value);
let (low, high) = fx.bcx.ins().isplit(bits);
let high = fx.bcx.ins().bxor_imm(high, 0x8000_0000_0000_0000_u64 as i64);
let bits = fx.bcx.ins().iconcat(low, high);
fx.bcx.ins().bitcast(types::F128, MemFlags::new(), bits)
}
pub(crate) fn abs_f16(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
let bits = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), value);
let bits = fx.bcx.ins().band_imm(bits, 0x7fff);
fx.bcx.ins().bitcast(types::F16, MemFlags::new(), bits)
}
pub(crate) fn abs_f128(fx: &mut FunctionCx<'_, '_, '_>, value: Value) -> Value {
let bits = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), value);
let (low, high) = fx.bcx.ins().isplit(bits);
let high = fx.bcx.ins().band_imm(high, 0x7fff_ffff_ffff_ffff_u64 as i64);
let bits = fx.bcx.ins().iconcat(low, high);
fx.bcx.ins().bitcast(types::F128, MemFlags::new(), bits)
}
pub(crate) fn copysign_f16(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value {
let lhs = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), lhs);
let rhs = fx.bcx.ins().bitcast(types::I16, MemFlags::new(), rhs);
let res = fx.bcx.ins().band_imm(lhs, 0x7fff);
let sign = fx.bcx.ins().band_imm(rhs, 0x8000);
let res = fx.bcx.ins().bor(res, sign);
fx.bcx.ins().bitcast(types::F16, MemFlags::new(), res)
}
pub(crate) fn copysign_f128(fx: &mut FunctionCx<'_, '_, '_>, lhs: Value, rhs: Value) -> Value {
let lhs = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), lhs);
let rhs = fx.bcx.ins().bitcast(types::I128, MemFlags::new(), rhs);
let (low, lhs_high) = fx.bcx.ins().isplit(lhs);
let (_, rhs_high) = fx.bcx.ins().isplit(rhs);
let high = fx.bcx.ins().band_imm(lhs_high, 0x7fff_ffff_ffff_ffff_u64 as i64);
let sign = fx.bcx.ins().band_imm(rhs_high, 0x8000_0000_0000_0000_u64 as i64);
let high = fx.bcx.ins().bor(high, sign);
let res = fx.bcx.ins().iconcat(low, high);
fx.bcx.ins().bitcast(types::F128, MemFlags::new(), res)
}
pub(crate) fn codegen_cast(
fx: &mut FunctionCx<'_, '_, '_>,
from: Value,
from_signed: bool,
to_ty: Type,
to_signed: bool,
) -> Value {
let from_ty = fx.bcx.func.dfg.value_type(from);
if from_ty.is_float() && to_ty.is_float() {
let name = match (from_ty, to_ty) {
(types::F16, types::F32) => return f16_to_f32(fx, from),
(types::F16, types::F64) => return f16_to_f64(fx, from),
(types::F16, types::F128) => "__extendhftf2",
(types::F32, types::F128) => "__extendsftf2",
(types::F64, types::F128) => "__extenddftf2",
(types::F128, types::F64) => "__trunctfdf2",
(types::F128, types::F32) => "__trunctfsf2",
(types::F128, types::F16) => "__trunctfhf2",
(types::F64, types::F16) => return f64_to_f16(fx, from),
(types::F32, types::F16) => return f32_to_f16(fx, from),
_ => unreachable!("{from_ty:?} -> {to_ty:?}"),
};
fx.lib_call(name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(to_ty)], &[from])[0]
} else if from_ty.is_int() && to_ty == types::F16 {
let res = clif_int_or_float_cast(fx, from, from_signed, types::F32, false);
f32_to_f16(fx, res)
} else if from_ty == types::F16 && to_ty.is_int() {
let from = f16_to_f32(fx, from);
clif_int_or_float_cast(fx, from, false, to_ty, to_signed)
} else if from_ty.is_int() && to_ty == types::F128 {
let (from, from_ty) = if from_ty.bits() < 32 {
(clif_int_or_float_cast(fx, from, from_signed, types::I32, from_signed), types::I32)
} else {
(from, from_ty)
};
let name = format!(
"__float{sign}{size}itf",
sign = if from_signed { "" } else { "un" },
size = match from_ty {
types::I32 => 's',
types::I64 => 'd',
types::I128 => 't',
_ => unreachable!("{from_ty:?}"),
},
);
fx.lib_call(
&name,
vec![lib_call_arg_param(fx.tcx, from_ty, from_signed)],
vec![AbiParam::new(to_ty)],
&[from],
)[0]
} else if from_ty == types::F128 && to_ty.is_int() {
let ret_ty = if to_ty.bits() < 32 { types::I32 } else { to_ty };
let name = format!(
"__fix{sign}tf{size}i",
sign = if from_signed { "" } else { "un" },
size = match ret_ty {
types::I32 => 's',
types::I64 => 'd',
types::I128 => 't',
_ => unreachable!("{from_ty:?}"),
},
);
let ret =
fx.lib_call(&name, vec![AbiParam::new(from_ty)], vec![AbiParam::new(to_ty)], &[from])
[0];
let val = if ret_ty == to_ty {
ret
} else {
let (min, max) = match (to_ty, to_signed) {
(types::I8, false) => (0, i64::from(u8::MAX)),
(types::I16, false) => (0, i64::from(u16::MAX)),
(types::I8, true) => (i64::from(i8::MIN as u32), i64::from(i8::MAX as u32)),
(types::I16, true) => (i64::from(i16::MIN as u32), i64::from(i16::MAX as u32)),
_ => unreachable!("{to_ty:?}"),
};
let min_val = fx.bcx.ins().iconst(types::I32, min);
let max_val = fx.bcx.ins().iconst(types::I32, max);
let val = if to_signed {
let has_underflow = fx.bcx.ins().icmp_imm(IntCC::SignedLessThan, ret, min);
let has_overflow = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThan, ret, max);
let bottom_capped = fx.bcx.ins().select(has_underflow, min_val, ret);
fx.bcx.ins().select(has_overflow, max_val, bottom_capped)
} else {
let has_overflow = fx.bcx.ins().icmp_imm(IntCC::UnsignedGreaterThan, ret, max);
fx.bcx.ins().select(has_overflow, max_val, ret)
};
fx.bcx.ins().ireduce(to_ty, val)
};
if let Some(false) = fx.tcx.sess.opts.unstable_opts.saturating_float_casts {
return val;
}
let is_not_nan = fcmp(fx, FloatCC::Equal, from, from);
let zero = type_zero_value(&mut fx.bcx, to_ty);
fx.bcx.ins().select(is_not_nan, val, zero)
} else {
unreachable!("{from_ty:?} -> {to_ty:?}");
}
}
pub(crate) fn fma_f16(fx: &mut FunctionCx<'_, '_, '_>, x: Value, y: Value, z: Value) -> Value {
let x = f16_to_f64(fx, x);
let y = f16_to_f64(fx, y);
let z = f16_to_f64(fx, z);
let res = fx.bcx.ins().fma(x, y, z);
f64_to_f16(fx, res)
}
pub(crate) fn fmin_f128(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
fx.lib_call(
"fminimumf128",
vec![AbiParam::new(types::F128), AbiParam::new(types::F128)],
vec![AbiParam::new(types::F128)],
&[a, b],
)[0]
}
pub(crate) fn fmax_f128(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
fx.lib_call(
"fmaximumf128",
vec![AbiParam::new(types::F128), AbiParam::new(types::F128)],
vec![AbiParam::new(types::F128)],
&[a, b],
)[0]
}

8
compiler/rustc_codegen_cranelift/src/common.rs
View file

@ -33,10 +33,10 @@ pub(crate) fn scalar_to_clif_type(tcx: TyCtxt<'_>, scalar: Scalar) -> Type {
Integer::I128 => types::I128,
},
Primitive::Float(float) => match float {
Float::F16 => unimplemented!("f16_f128"),
Float::F16 => types::F16,
Float::F32 => types::F32,
Float::F64 => types::F64,
Float::F128 => unimplemented!("f16_f128"),
Float::F128 => types::F128,
},
// FIXME(erikdesjardins): handle non-default addrspace ptr sizes
Primitive::Pointer(_) => pointer_ty(tcx),
@ -64,10 +64,10 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Typ
},
ty::Char => types::I32,
ty::Float(size) => match size {
FloatTy::F16 => unimplemented!("f16_f128"),
FloatTy::F16 => types::F16,
FloatTy::F32 => types::F32,
FloatTy::F64 => types::F64,
FloatTy::F128 => unimplemented!("f16_f128"),
FloatTy::F128 => types::F128,
},
ty::FnPtr(..) => pointer_ty(tcx),
ty::RawPtr(pointee_ty, _) | ty::Ref(_, pointee_ty, _) => {

88
compiler/rustc_codegen_cranelift/src/compiler_builtins.rs
View file

@ -46,15 +46,100 @@ builtin_functions! {
fn __rust_u128_mulo(a: u128, b: u128, oflow: &mut i32) -> u128;
fn __rust_i128_mulo(a: i128, b: i128, oflow: &mut i32) -> i128;
// floats
// integer -> float
fn __floattisf(i: i128) -> f32;
fn __floattidf(i: i128) -> f64;
fn __floatsitf(i: i32) -> f128;
fn __floatditf(i: i64) -> f128;
fn __floattitf(i: i128) -> f128;
fn __floatuntisf(i: u128) -> f32;
fn __floatuntidf(i: u128) -> f64;
fn __floatunsitf(i: u32) -> f128;
fn __floatunditf(i: u64) -> f128;
fn __floatuntitf(i: u128) -> f128;
// float -> integer
fn __fixsfti(f: f32) -> i128;
fn __fixdfti(f: f64) -> i128;
fn __fixtfsi(f: f128) -> i32;
fn __fixtfdi(f: f128) -> i64;
fn __fixtfti(f: f128) -> i128;
fn __fixunssfti(f: f32) -> u128;
fn __fixunsdfti(f: f64) -> u128;
fn __fixunstfsi(f: f128) -> u32;
fn __fixunstfdi(f: f128) -> u64;
fn __fixunstfti(f: f128) -> u128;
// float -> float
fn __extendhfsf2(f: f16) -> f32;
fn __extendhftf2(f: f16) -> f128;
fn __extendsftf2(f: f32) -> f128;
fn __extenddftf2(f: f64) -> f128;
fn __trunctfdf2(f: f128) -> f64;
fn __trunctfsf2(f: f128) -> f32;
fn __trunctfhf2(f: f128) -> f16;
fn __truncdfhf2(f: f64) -> f16;
fn __truncsfhf2(f: f32) -> f16;
// float binops
fn __addtf3(a: f128, b: f128) -> f128;
fn __subtf3(a: f128, b: f128) -> f128;
fn __multf3(a: f128, b: f128) -> f128;
fn __divtf3(a: f128, b: f128) -> f128;
fn fmodf(a: f32, b: f32) -> f32;
fn fmod(a: f64, b: f64) -> f64;
fn fmodf128(a: f128, b: f128) -> f128;
// float comparison
fn __eqtf2(a: f128, b: f128) -> i32;
fn __netf2(a: f128, b: f128) -> i32;
fn __lttf2(a: f128, b: f128) -> i32;
fn __letf2(a: f128, b: f128) -> i32;
fn __gttf2(a: f128, b: f128) -> i32;
fn __getf2(a: f128, b: f128) -> i32;
fn fminimumf128(a: f128, b: f128) -> f128;
fn fmaximumf128(a: f128, b: f128) -> f128;
// Cranelift float libcalls
fn fmaf(a: f32, b: f32, c: f32) -> f32;
fn fma(a: f64, b: f64, c: f64) -> f64;
fn floorf(f: f32) -> f32;
fn floor(f: f64) -> f64;
fn ceilf(f: f32) -> f32;
fn ceil(f: f64) -> f64;
fn truncf(f: f32) -> f32;
fn trunc(f: f64) -> f64;
fn nearbyintf(f: f32) -> f32;
fn nearbyint(f: f64) -> f64;
// float intrinsics
fn __powisf2(a: f32, b: i32) -> f32;
fn __powidf2(a: f64, b: i32) -> f64;
// FIXME(f16_f128): `compiler-builtins` doesn't currently support `__powitf2` on MSVC.
// fn __powitf2(a: f128, b: i32) -> f128;
fn powf(a: f32, b: f32) -> f32;
fn pow(a: f64, b: f64) -> f64;
fn expf(f: f32) -> f32;
fn exp(f: f64) -> f64;
fn exp2f(f: f32) -> f32;
fn exp2(f: f64) -> f64;
fn logf(f: f32) -> f32;
fn log(f: f64) -> f64;
fn log2f(f: f32) -> f32;
fn log2(f: f64) -> f64;
fn log10f(f: f32) -> f32;
fn log10(f: f64) -> f64;
fn sinf(f: f32) -> f32;
fn sin(f: f64) -> f64;
fn cosf(f: f32) -> f32;
fn cos(f: f64) -> f64;
fn fmaf128(a: f128, b: f128, c: f128) -> f128;
fn floorf16(f: f16) -> f16;
fn floorf128(f: f128) -> f128;
fn ceilf16(f: f16) -> f16;
fn ceilf128(f: f128) -> f128;
fn truncf16(f: f16) -> f16;
fn truncf128(f: f128) -> f128;
fn rintf16(f: f16) -> f16;
fn rintf128(f: f128) -> f128;
fn sqrtf16(f: f16) -> f16;
fn sqrtf128(f: f128) -> f128;
// FIXME(f16_f128): Add other float intrinsics as compiler-builtins gains support (meaning they
// are available on all targets).
// allocator
// NOTE: These need to be mentioned here despite not being part of compiler_builtins because
@ -67,5 +152,4 @@ builtin_functions! {
fn malloc(size: size_t) -> *mut c_void;
fn realloc(p: *mut c_void, size: size_t) -> *mut c_void;
fn free(p: *mut c_void) -> ();
}

39
compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
View file

@ -81,13 +81,36 @@ impl WriterRelocate {
/// Perform the collected relocations to be usable for JIT usage.
#[cfg(all(feature = "jit", not(windows)))]
pub(super) fn relocate_for_jit(mut self, jit_module: &cranelift_jit::JITModule) -> Vec<u8> {
use cranelift_module::Module;
for reloc in self.relocs.drain(..) {
match reloc.name {
super::DebugRelocName::Section(_) => unreachable!(),
super::DebugRelocName::Symbol(sym) => {
let addr = jit_module.get_finalized_function(
cranelift_module::FuncId::from_u32(sym.try_into().unwrap()),
);
let addr = if sym & 1 << 31 == 0 {
let func_id = FuncId::from_u32(sym.try_into().unwrap());
// FIXME make JITModule::get_address public and use it here instead.
// HACK rust_eh_personality is likely not defined in the same crate,
// so get_finalized_function won't work. Use the rust_eh_personality
// of cg_clif itself, which is likely ABI compatible.
if jit_module.declarations().get_function_decl(func_id).name.as_deref()
== Some("rust_eh_personality")
{
extern "C" {
fn rust_eh_personality() -> !;
}
rust_eh_personality as *const u8
} else {
jit_module.get_finalized_function(func_id)
}
} else {
jit_module
.get_finalized_data(DataId::from_u32(
u32::try_from(sym).unwrap() & !(1 << 31),
))
.0
};
let val = (addr as u64 as i64 + reloc.addend) as u64;
self.writer.write_udata_at(reloc.offset as usize, val, reloc.size).unwrap();
}
@ -196,6 +219,16 @@ impl Writer for WriterRelocate {
});
self.write_udata(0, size)
}
gimli::DW_EH_PE_absptr => {
self.relocs.push(DebugReloc {
offset: self.len() as u32,
size: size.into(),
name: DebugRelocName::Symbol(symbol),
addend,
kind: object::RelocationKind::Absolute,
});
self.write_udata(0, size.into())
}
_ => Err(gimli::write::Error::UnsupportedPointerEncoding(eh_pe)),
},
}

20
compiler/rustc_codegen_cranelift/src/debuginfo/unwind.rs
View file

@ -1,7 +1,6 @@
//! Unwind info generation (`.eh_frame`)
use cranelift_codegen::ir::Endianness;
use cranelift_codegen::isa::TargetIsa;
use cranelift_codegen::isa::unwind::UnwindInfo;
use cranelift_object::ObjectProduct;
use gimli::RunTimeEndian;
@ -18,14 +17,14 @@ pub(crate) struct UnwindContext {
}
impl UnwindContext {
pub(crate) fn new(isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self {
let endian = match isa.endianness() {
pub(crate) fn new(module: &mut dyn Module, pic_eh_frame: bool) -> Self {
let endian = match module.isa().endianness() {
Endianness::Little => RunTimeEndian::Little,
Endianness::Big => RunTimeEndian::Big,
};
let mut frame_table = FrameTable::default();
let cie_id = if let Some(mut cie) = isa.create_systemv_cie() {
let cie_id = if let Some(mut cie) = module.isa().create_systemv_cie() {
if pic_eh_frame {
cie.fde_address_encoding =
gimli::DwEhPe(gimli::DW_EH_PE_pcrel.0 | gimli::DW_EH_PE_sdata4.0);
@ -38,8 +37,15 @@ impl UnwindContext {
UnwindContext { endian, frame_table, cie_id }
}
pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) {
if let target_lexicon::OperatingSystem::MacOSX { .. } = isa.triple().operating_system {
pub(crate) fn add_function(
&mut self,
module: &mut dyn Module,
func_id: FuncId,
context: &Context,
) {
if let target_lexicon::OperatingSystem::MacOSX { .. } =
module.isa().triple().operating_system
{
// The object crate doesn't currently support DW_GNU_EH_PE_absptr, which macOS
// requires for unwinding tables. In addition on arm64 it currently doesn't
// support 32bit relocations as we currently use for the unwinding table.
@ -48,7 +54,7 @@ impl UnwindContext {
}
let unwind_info = if let Some(unwind_info) =
context.compiled_code().unwrap().create_unwind_info(isa).unwrap()
context.compiled_code().unwrap().create_unwind_info(module.isa()).unwrap()
{
unwind_info
} else {

7
compiler/rustc_codegen_cranelift/src/driver/jit.rs
View file

@ -6,6 +6,7 @@ use std::os::raw::{c_char, c_int};
use cranelift_jit::{JITBuilder, JITModule};
use rustc_codegen_ssa::CrateInfo;
use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
use rustc_middle::mir::mono::MonoItem;
use rustc_session::Session;
use rustc_span::sym;
@ -84,7 +85,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, jit_args: Vec<String>) -> ! {
tcx.dcx().abort_if_errors();
jit_module.finalize_definitions();
let mut jit_module = jit_module.finalize_definitions();
println!(
"Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed"
@ -104,7 +105,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, jit_args: Vec<String>) -> ! {
call_conv: jit_module.target_config().default_call_conv,
};
let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap();
let finalized_start: *const u8 = jit_module.module.get_finalized_function(start_func_id);
let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id);
let f: extern "C" fn(c_int, *const *const c_char) -> c_int =
unsafe { ::std::mem::transmute(finalized_start) };
@ -119,7 +120,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, jit_args: Vec<String>) -> ! {
std::process::exit(ret);
}
pub(crate) fn codegen_and_compile_fn<'tcx>(
fn codegen_and_compile_fn<'tcx>(
tcx: TyCtxt<'tcx>,
cx: &mut crate::CodegenCx,
cached_context: &mut Context,

7
compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
View file

@ -66,7 +66,12 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
fx.tcx
.dcx()
.warn(format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic));
crate::trap::trap_unimplemented(fx, intrinsic);
let msg = format!(
"{intrinsic} is not yet supported.\n\
See https://github.com/rust-lang/rustc_codegen_cranelift/issues/171\n\
Please open an issue at https://github.com/rust-lang/rustc_codegen_cranelift/issues"
);
crate::base::codegen_panic_nounwind(fx, &msg, None);
return;
}
}

9
compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs
View file

@ -7,7 +7,7 @@ use crate::inline_asm::{CInlineAsmOperand, codegen_inline_asm_inner};
use crate::intrinsics::*;
use crate::prelude::*;
pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>(
pub(super) fn codegen_aarch64_llvm_intrinsic_call<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
intrinsic: &str,
args: &[Spanned<mir::Operand<'tcx>>],
@ -507,7 +507,12 @@ pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>(
"unsupported AArch64 llvm intrinsic {}; replacing with trap",
intrinsic
));
crate::trap::trap_unimplemented(fx, intrinsic);
let msg = format!(
"{intrinsic} is not yet supported.\n\
See https://github.com/rust-lang/rustc_codegen_cranelift/issues/171\n\
Please open an issue at https://github.com/rust-lang/rustc_codegen_cranelift/issues"
);
crate::base::codegen_panic_nounwind(fx, &msg, None);
return;
}
}

13
compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
View file

@ -7,7 +7,7 @@ use crate::inline_asm::{CInlineAsmOperand, codegen_inline_asm_inner};
use crate::intrinsics::*;
use crate::prelude::*;
pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
pub(super) fn codegen_x86_llvm_intrinsic_call<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
intrinsic: &str,
args: &[Spanned<mir::Operand<'tcx>>],
@ -147,10 +147,10 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
let offset = fx.bcx.ins().imul(index_lane, scale);
let lane_ptr = fx.bcx.ins().iadd(ptr, offset);
let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), lane_ptr, 0);
fx.bcx.ins().jump(next, &[res]);
fx.bcx.ins().jump(next, &[res.into()]);
fx.bcx.switch_to_block(if_disabled);
fx.bcx.ins().jump(next, &[src_lane]);
fx.bcx.ins().jump(next, &[src_lane.into()]);
fx.bcx.seal_block(next);
fx.bcx.switch_to_block(next);
@ -1316,7 +1316,12 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>(
fx.tcx
.dcx()
.warn(format!("unsupported x86 llvm intrinsic {}; replacing with trap", intrinsic));
crate::trap::trap_unimplemented(fx, intrinsic);
let msg = format!(
"{intrinsic} is not yet supported.\n\
See https://github.com/rust-lang/rustc_codegen_cranelift/issues/171\n\
Please open an issue at https://github.com/rust-lang/rustc_codegen_cranelift/issues"
);
crate::base::codegen_panic_nounwind(fx, &msg, None);
return;
}
}

181
compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View file

@ -27,6 +27,7 @@ use rustc_span::{Symbol, sym};
pub(crate) use self::llvm::codegen_llvm_intrinsic_call;
use crate::cast::clif_intcast;
use crate::codegen_f16_f128;
use crate::prelude::*;
fn bug_on_incorrect_arg_count(intrinsic: impl std::fmt::Display) -> ! {
@ -248,8 +249,10 @@ fn bool_to_zero_or_max_uint<'tcx>(
let ty = fx.clif_type(ty).unwrap();
let int_ty = match ty {
types::F16 => types::I16,
types::F32 => types::I32,
types::F64 => types::I64,
types::F128 => types::I128,
ty => ty,
};
@ -308,45 +311,83 @@ fn codegen_float_intrinsic_call<'tcx>(
ret: CPlace<'tcx>,
) -> bool {
let (name, arg_count, ty, clif_ty) = match intrinsic {
sym::expf16 => ("expf16", 1, fx.tcx.types.f16, types::F16),
sym::expf32 => ("expf", 1, fx.tcx.types.f32, types::F32),
sym::expf64 => ("exp", 1, fx.tcx.types.f64, types::F64),
sym::expf128 => ("expf128", 1, fx.tcx.types.f128, types::F128),
sym::exp2f16 => ("exp2f16", 1, fx.tcx.types.f16, types::F16),
sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32, types::F32),
sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64, types::F64),
sym::exp2f128 => ("exp2f128", 1, fx.tcx.types.f128, types::F128),
sym::sqrtf16 => ("sqrtf16", 1, fx.tcx.types.f16, types::F16),
sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32, types::F32),
sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64, types::F64),
sym::sqrtf128 => ("sqrtf128", 1, fx.tcx.types.f128, types::F128),
sym::powif16 => ("__powisf2", 2, fx.tcx.types.f16, types::F16), // compiler-builtins
sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32, types::F32), // compiler-builtins
sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64, types::F64), // compiler-builtins
sym::powif128 => ("__powitf2", 2, fx.tcx.types.f128, types::F128), // compiler-builtins
sym::powf16 => ("powf16", 2, fx.tcx.types.f16, types::F16),
sym::powf32 => ("powf", 2, fx.tcx.types.f32, types::F32),
sym::powf64 => ("pow", 2, fx.tcx.types.f64, types::F64),
sym::powf128 => ("powf128", 2, fx.tcx.types.f128, types::F128),
sym::logf16 => ("logf16", 1, fx.tcx.types.f16, types::F16),
sym::logf32 => ("logf", 1, fx.tcx.types.f32, types::F32),
sym::logf64 => ("log", 1, fx.tcx.types.f64, types::F64),
sym::logf128 => ("logf128", 1, fx.tcx.types.f128, types::F128),
sym::log2f16 => ("log2f16", 1, fx.tcx.types.f16, types::F16),
sym::log2f32 => ("log2f", 1, fx.tcx.types.f32, types::F32),
sym::log2f64 => ("log2", 1, fx.tcx.types.f64, types::F64),
sym::log2f128 => ("log2f128", 1, fx.tcx.types.f128, types::F128),
sym::log10f16 => ("log10f16", 1, fx.tcx.types.f16, types::F16),
sym::log10f32 => ("log10f", 1, fx.tcx.types.f32, types::F32),
sym::log10f64 => ("log10", 1, fx.tcx.types.f64, types::F64),
sym::log10f128 => ("log10f128", 1, fx.tcx.types.f128, types::F128),
sym::fabsf16 => ("fabsf16", 1, fx.tcx.types.f16, types::F16),
sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32, types::F32),
sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64, types::F64),
sym::fabsf128 => ("fabsf128", 1, fx.tcx.types.f128, types::F128),
sym::fmaf16 => ("fmaf16", 3, fx.tcx.types.f16, types::F16),
sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32),
sym::fmaf64 => ("fma", 3, fx.tcx.types.f64, types::F64),
sym::fmaf128 => ("fmaf128", 3, fx.tcx.types.f128, types::F128),
// FIXME: calling `fma` from libc without FMA target feature uses expensive sofware emulation
sym::fmuladdf16 => ("fmaf16", 3, fx.tcx.types.f16, types::F16), // TODO: use cranelift intrinsic analogous to llvm.fmuladd.f16
sym::fmuladdf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32), // TODO: use cranelift intrinsic analogous to llvm.fmuladd.f32
sym::fmuladdf64 => ("fma", 3, fx.tcx.types.f64, types::F64), // TODO: use cranelift intrinsic analogous to llvm.fmuladd.f64
sym::fmuladdf128 => ("fmaf128", 3, fx.tcx.types.f128, types::F128), // TODO: use cranelift intrinsic analogous to llvm.fmuladd.f128
sym::copysignf16 => ("copysignf16", 2, fx.tcx.types.f16, types::F16),
sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32, types::F32),
sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64, types::F64),
sym::copysignf128 => ("copysignf128", 2, fx.tcx.types.f128, types::F128),
sym::floorf16 => ("floorf16", 1, fx.tcx.types.f16, types::F16),
sym::floorf32 => ("floorf", 1, fx.tcx.types.f32, types::F32),
sym::floorf64 => ("floor", 1, fx.tcx.types.f64, types::F64),
sym::floorf128 => ("floorf128", 1, fx.tcx.types.f128, types::F128),
sym::ceilf16 => ("ceilf16", 1, fx.tcx.types.f16, types::F16),
sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32, types::F32),
sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64, types::F64),
sym::ceilf128 => ("ceilf128", 1, fx.tcx.types.f128, types::F128),
sym::truncf16 => ("truncf16", 1, fx.tcx.types.f16, types::F16),
sym::truncf32 => ("truncf", 1, fx.tcx.types.f32, types::F32),
sym::truncf64 => ("trunc", 1, fx.tcx.types.f64, types::F64),
sym::truncf128 => ("truncf128", 1, fx.tcx.types.f128, types::F128),
sym::round_ties_even_f16 => ("rintf16", 1, fx.tcx.types.f16, types::F16),
sym::round_ties_even_f32 => ("rintf", 1, fx.tcx.types.f32, types::F32),
sym::round_ties_even_f64 => ("rint", 1, fx.tcx.types.f64, types::F64),
sym::round_ties_even_f128 => ("rintf128", 1, fx.tcx.types.f128, types::F128),
sym::roundf16 => ("roundf16", 1, fx.tcx.types.f16, types::F16),
sym::roundf32 => ("roundf", 1, fx.tcx.types.f32, types::F32),
sym::roundf64 => ("round", 1, fx.tcx.types.f64, types::F64),
sym::roundf128 => ("roundf128", 1, fx.tcx.types.f128, types::F128),
sym::sinf16 => ("sinf16", 1, fx.tcx.types.f16, types::F16),
sym::sinf32 => ("sinf", 1, fx.tcx.types.f32, types::F32),
sym::sinf64 => ("sin", 1, fx.tcx.types.f64, types::F64),
sym::sinf128 => ("sinf128", 1, fx.tcx.types.f128, types::F128),
sym::cosf16 => ("cosf16", 1, fx.tcx.types.f16, types::F16),
sym::cosf32 => ("cosf", 1, fx.tcx.types.f32, types::F32),
sym::cosf64 => ("cos", 1, fx.tcx.types.f64, types::F64),
sym::cosf128 => ("cosf128", 1, fx.tcx.types.f128, types::F128),
_ => return false,
};
@ -379,13 +420,26 @@ fn codegen_float_intrinsic_call<'tcx>(
};
let layout = fx.layout_of(ty);
// FIXME(bytecodealliance/wasmtime#8312): Use native Cranelift operations
// for `f16` and `f128` once the lowerings have been implemented in Cranelift.
let res = match intrinsic {
sym::fmaf16 | sym::fmuladdf16 => {
CValue::by_val(codegen_f16_f128::fma_f16(fx, args[0], args[1], args[2]), layout)
}
sym::fmaf32 | sym::fmaf64 | sym::fmuladdf32 | sym::fmuladdf64 => {
CValue::by_val(fx.bcx.ins().fma(args[0], args[1], args[2]), layout)
}
sym::copysignf16 => {
CValue::by_val(codegen_f16_f128::copysign_f16(fx, args[0], args[1]), layout)
}
sym::copysignf128 => {
CValue::by_val(codegen_f16_f128::copysign_f128(fx, args[0], args[1]), layout)
}
sym::copysignf32 | sym::copysignf64 => {
CValue::by_val(fx.bcx.ins().fcopysign(args[0], args[1]), layout)
}
sym::fabsf16 => CValue::by_val(codegen_f16_f128::abs_f16(fx, args[0]), layout),
sym::fabsf128 => CValue::by_val(codegen_f16_f128::abs_f128(fx, args[0]), layout),
sym::fabsf32
| sym::fabsf64
| sym::floorf32
@ -415,11 +469,36 @@ fn codegen_float_intrinsic_call<'tcx>(
// These intrinsics aren't supported natively by Cranelift.
// Lower them to a libcall.
sym::powif32 | sym::powif64 => {
let input_tys: Vec<_> = vec![AbiParam::new(clif_ty), AbiParam::new(types::I32)];
sym::powif16 | sym::powif32 | sym::powif64 | sym::powif128 => {
let temp;
let (clif_ty, args) = if intrinsic == sym::powif16 {
temp = [codegen_f16_f128::f16_to_f32(fx, args[0]), args[1]];
(types::F32, temp.as_slice())
} else {
(clif_ty, args)
};
let input_tys: Vec<_> =
vec![AbiParam::new(clif_ty), lib_call_arg_param(fx.tcx, types::I32, true)];
let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0];
let ret_val = if intrinsic == sym::powif16 {
codegen_f16_f128::f32_to_f16(fx, ret_val)
} else {
ret_val
};
CValue::by_val(ret_val, fx.layout_of(ty))
}
sym::powf16 => {
// FIXME(f16_f128): Rust `compiler-builtins` doesn't export `powf16` yet.
let x = codegen_f16_f128::f16_to_f32(fx, args[0]);
let y = codegen_f16_f128::f16_to_f32(fx, args[1]);
let ret_val = fx.lib_call(
"powf",
vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
vec![AbiParam::new(types::F32)],
&[x, y],
)[0];
CValue::by_val(codegen_f16_f128::f32_to_f16(fx, ret_val), fx.layout_of(ty))
}
_ => {
let input_tys: Vec<_> = args.iter().map(|_| AbiParam::new(clif_ty)).collect();
let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0];
@ -801,7 +880,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
// FIXME implement 128bit atomics
if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
// special case for compiler-builtins to avoid having to patch it
crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
crate::base::codegen_panic_nounwind(
fx,
"128bit atomics not yet supported",
None,
);
return Ok(());
} else {
fx.tcx
@ -832,7 +915,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
// FIXME implement 128bit atomics
if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
// special case for compiler-builtins to avoid having to patch it
crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
crate::base::codegen_panic_nounwind(
fx,
"128bit atomics not yet supported",
None,
);
return Ok(());
} else {
fx.tcx
@ -1109,6 +1196,20 @@ fn codegen_regular_intrinsic_call<'tcx>(
ret.write_cvalue(fx, old);
}
sym::minimumf16 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
let b = b.load_scalar(fx);
// FIXME(bytecodealliance/wasmtime#8312): Use `fmin` directly once
// Cranelift backend lowerings are implemented.
let a = codegen_f16_f128::f16_to_f32(fx, a);
let b = codegen_f16_f128::f16_to_f32(fx, b);
let val = fx.bcx.ins().fmin(a, b);
let val = codegen_f16_f128::f32_to_f16(fx, val);
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f16));
ret.write_cvalue(fx, val);
}
sym::minimumf32 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
@ -1127,6 +1228,31 @@ fn codegen_regular_intrinsic_call<'tcx>(
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
ret.write_cvalue(fx, val);
}
sym::minimumf128 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
let b = b.load_scalar(fx);
// FIXME(bytecodealliance/wasmtime#8312): Use `fmin` once Cranelift
// backend lowerings are implemented.
let val = codegen_f16_f128::fmin_f128(fx, a, b);
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f128));
ret.write_cvalue(fx, val);
}
sym::maximumf16 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
let b = b.load_scalar(fx);
// FIXME(bytecodealliance/wasmtime#8312): Use `fmax` directly once
// Cranelift backend lowerings are implemented.
let a = codegen_f16_f128::f16_to_f32(fx, a);
let b = codegen_f16_f128::f16_to_f32(fx, b);
let val = fx.bcx.ins().fmax(a, b);
let val = codegen_f16_f128::f32_to_f16(fx, val);
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f16));
ret.write_cvalue(fx, val);
}
sym::maximumf32 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
@ -1145,7 +1271,27 @@ fn codegen_regular_intrinsic_call<'tcx>(
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
ret.write_cvalue(fx, val);
}
sym::maximumf128 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
let b = b.load_scalar(fx);
// FIXME(bytecodealliance/wasmtime#8312): Use `fmax` once Cranelift
// backend lowerings are implemented.
let val = codegen_f16_f128::fmax_f128(fx, a, b);
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f128));
ret.write_cvalue(fx, val);
}
sym::minnumf16 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
let b = b.load_scalar(fx);
let val = crate::num::codegen_float_min(fx, a, b);
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f16));
ret.write_cvalue(fx, val);
}
sym::minnumf32 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
@ -1164,6 +1310,24 @@ fn codegen_regular_intrinsic_call<'tcx>(
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
ret.write_cvalue(fx, val);
}
sym::minnumf128 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
let b = b.load_scalar(fx);
let val = crate::num::codegen_float_min(fx, a, b);
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f128));
ret.write_cvalue(fx, val);
}
sym::maxnumf16 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
let b = b.load_scalar(fx);
let val = crate::num::codegen_float_max(fx, a, b);
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f16));
ret.write_cvalue(fx, val);
}
sym::maxnumf32 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
@ -1182,6 +1346,15 @@ fn codegen_regular_intrinsic_call<'tcx>(
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
ret.write_cvalue(fx, val);
}
sym::maxnumf128 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
let b = b.load_scalar(fx);
let val = crate::num::codegen_float_max(fx, a, b);
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f128));
ret.write_cvalue(fx, val);
}
sym::catch_unwind => {
intrinsic_args!(fx, args => (f, data, catch_fn); intrinsic);

36
compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
View file

@ -283,6 +283,20 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
ret_lane.write_cvalue(fx, val);
}
sym::simd_insert_dyn => {
intrinsic_args!(fx, args => (base, idx, val); intrinsic);
if !base.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, base.layout().ty);
return;
}
let idx = idx.load_scalar(fx);
ret.write_cvalue(fx, base);
ret.write_lane_dyn(fx, idx, val);
}
sym::simd_extract => {
let (v, idx) = match args {
[v, idx] => (v, idx),
@ -318,6 +332,20 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
ret.write_cvalue(fx, ret_lane);
}
sym::simd_extract_dyn => {
intrinsic_args!(fx, args => (v, idx); intrinsic);
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
let idx = idx.load_scalar(fx);
let ret_lane = v.value_lane_dyn(fx, idx);
ret.write_cvalue(fx, ret_lane);
}
sym::simd_neg
| sym::simd_bswap
| sym::simd_bitreverse
@ -980,10 +1008,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
fx.bcx.switch_to_block(if_enabled);
let res = fx.bcx.ins().load(lane_clif_ty, MemFlags::trusted(), ptr_lane, 0);
fx.bcx.ins().jump(next, &[res]);
fx.bcx.ins().jump(next, &[res.into()]);
fx.bcx.switch_to_block(if_disabled);
fx.bcx.ins().jump(next, &[val_lane]);
fx.bcx.ins().jump(next, &[val_lane.into()]);
fx.bcx.seal_block(next);
fx.bcx.switch_to_block(next);
@ -1029,10 +1057,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
ptr_val,
Offset32::new(offset),
);
fx.bcx.ins().jump(next, &[res]);
fx.bcx.ins().jump(next, &[res.into()]);
fx.bcx.switch_to_block(if_disabled);
fx.bcx.ins().jump(next, &[val_lane]);
fx.bcx.ins().jump(next, &[val_lane.into()]);
fx.bcx.seal_block(next);
fx.bcx.switch_to_block(next);

43
compiler/rustc_codegen_cranelift/src/lib.rs
View file

@ -6,6 +6,9 @@
#![cfg_attr(doc, feature(rustdoc_internals))]
// Note: please avoid adding other feature gates where possible
#![feature(rustc_private)]
// Only used to define intrinsics in `compiler_builtins.rs`.
#![feature(f16)]
#![feature(f128)]
// Note: please avoid adding other feature gates where possible
#![warn(rust_2018_idioms)]
#![warn(unreachable_pub)]
@ -57,6 +60,7 @@ mod allocator;
mod analyze;
mod base;
mod cast;
mod codegen_f16_f128;
mod codegen_i128;
mod common;
mod compiler_builtins;
@ -76,7 +80,6 @@ mod optimize;
mod pointer;
mod pretty_clif;
mod toolchain;
mod trap;
mod unsize;
mod unwind_module;
mod value_and_place;
@ -198,14 +201,36 @@ impl CodegenBackend for CraneliftCodegenBackend {
// FIXME do `unstable_target_features` properly
let unstable_target_features = target_features.clone();
// FIXME(f16_f128): LLVM 20 (currently used by `rustc`) passes `f128` in XMM registers on
// Windows, whereas LLVM 21+ and Cranelift pass it indirectly. This means that `f128` won't
// work when linking against a LLVM-built sysroot.
let has_reliable_f128 = !sess.target.is_like_windows;
let has_reliable_f16 = match &*sess.target.arch {
// FIXME(f16_f128): LLVM 20 does not support `f16` on s390x, meaning the required
// builtins are not available in `compiler-builtins`.
"s390x" => false,
// FIXME(f16_f128): `rustc_codegen_llvm` currently disables support on Windows GNU
// targets due to GCC using a different ABI than LLVM. Therefore `f16` won't be
// available when using a LLVM-built sysroot.
"x86_64"
if sess.target.os == "windows"
&& sess.target.env == "gnu"
&& sess.target.abi != "llvm" =>
{
false
}
_ => true,
};
TargetConfig {
target_features,
unstable_target_features,
// Cranelift does not yet support f16 or f128
has_reliable_f16: false,
has_reliable_f16_math: false,
has_reliable_f128: false,
has_reliable_f128_math: false,
// `rustc_codegen_cranelift` polyfills functionality not yet
// available in Cranelift.
has_reliable_f16,
has_reliable_f16_math: has_reliable_f16,
has_reliable_f128,
has_reliable_f128_math: has_reliable_f128,
}
}
@ -290,6 +315,12 @@ fn build_isa(sess: &Session, jit: bool) -> Arc<dyn TargetIsa + 'static> {
flags_builder.set("enable_llvm_abi_extensions", "true").unwrap();
if let Some(align) = sess.opts.unstable_opts.min_function_alignment {
flags_builder
.set("log2_min_function_alignment", &align.bytes().ilog2().to_string())
.unwrap();
}
use rustc_session::config::OptLevel;
match sess.opts.optimize {
OptLevel::No => {

76
compiler/rustc_codegen_cranelift/src/num.rs
View file

@ -1,8 +1,9 @@
//! Various operations on integer and floating-point numbers
use crate::codegen_f16_f128;
use crate::prelude::*;
pub(crate) fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> IntCC {
fn bin_op_to_intcc(bin_op: BinOp, signed: bool) -> IntCC {
use BinOp::*;
use IntCC::*;
match bin_op {
@ -109,7 +110,7 @@ pub(crate) fn codegen_binop<'tcx>(
}
}
pub(crate) fn codegen_bool_binop<'tcx>(
fn codegen_bool_binop<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
bin_op: BinOp,
in_lhs: CValue<'tcx>,
@ -350,25 +351,60 @@ pub(crate) fn codegen_float_binop<'tcx>(
let lhs = in_lhs.load_scalar(fx);
let rhs = in_rhs.load_scalar(fx);
// FIXME(bytecodealliance/wasmtime#8312): Remove once backend lowerings have
// been added to Cranelift.
let (lhs, rhs) = if *in_lhs.layout().ty.kind() == ty::Float(FloatTy::F16) {
(codegen_f16_f128::f16_to_f32(fx, lhs), codegen_f16_f128::f16_to_f32(fx, rhs))
} else {
(lhs, rhs)
};
let b = fx.bcx.ins();
let res = match bin_op {
// FIXME(bytecodealliance/wasmtime#8312): Remove once backend lowerings
// have been added to Cranelift.
BinOp::Add | BinOp::Sub | BinOp::Mul | BinOp::Div
if *in_lhs.layout().ty.kind() == ty::Float(FloatTy::F128) =>
{
codegen_f16_f128::codegen_f128_binop(fx, bin_op, lhs, rhs)
}
BinOp::Add => b.fadd(lhs, rhs),
BinOp::Sub => b.fsub(lhs, rhs),
BinOp::Mul => b.fmul(lhs, rhs),
BinOp::Div => b.fdiv(lhs, rhs),
BinOp::Rem => {
let (name, ty) = match in_lhs.layout().ty.kind() {
ty::Float(FloatTy::F32) => ("fmodf", types::F32),
ty::Float(FloatTy::F64) => ("fmod", types::F64),
let (name, ty, lhs, rhs) = match in_lhs.layout().ty.kind() {
ty::Float(FloatTy::F16) => (
"fmodf",
types::F32,
// FIXME(bytecodealliance/wasmtime#8312): Already converted
// by the FIXME above.
// fx.bcx.ins().fpromote(types::F32, lhs),
// fx.bcx.ins().fpromote(types::F32, rhs),
lhs,
rhs,
),
ty::Float(FloatTy::F32) => ("fmodf", types::F32, lhs, rhs),
ty::Float(FloatTy::F64) => ("fmod", types::F64, lhs, rhs),
ty::Float(FloatTy::F128) => ("fmodf128", types::F128, lhs, rhs),
_ => bug!(),
};
fx.lib_call(
let ret_val = fx.lib_call(
name,
vec![AbiParam::new(ty), AbiParam::new(ty)],
vec![AbiParam::new(ty)],
&[lhs, rhs],
)[0]
)[0];
let ret_val = if *in_lhs.layout().ty.kind() == ty::Float(FloatTy::F16) {
// FIXME(bytecodealliance/wasmtime#8312): Use native Cranelift
// operation once Cranelift backend lowerings have been
// implemented.
codegen_f16_f128::f32_to_f16(fx, ret_val)
} else {
ret_val
};
return CValue::by_val(ret_val, in_lhs.layout());
}
BinOp::Eq | BinOp::Lt | BinOp::Le | BinOp::Ne | BinOp::Ge | BinOp::Gt => {
let fltcc = match bin_op {
@ -380,16 +416,26 @@ pub(crate) fn codegen_float_binop<'tcx>(
BinOp::Gt => FloatCC::GreaterThan,
_ => unreachable!(),
};
let val = fx.bcx.ins().fcmp(fltcc, lhs, rhs);
// FIXME(bytecodealliance/wasmtime#8312): Replace with Cranelift
// `fcmp` once `f16`/`f128` backend lowerings have been added to
// Cranelift.
let val = codegen_f16_f128::fcmp(fx, fltcc, lhs, rhs);
return CValue::by_val(val, fx.layout_of(fx.tcx.types.bool));
}
_ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs),
};
// FIXME(bytecodealliance/wasmtime#8312): Remove once backend lowerings have
// been added to Cranelift.
let res = if *in_lhs.layout().ty.kind() == ty::Float(FloatTy::F16) {
codegen_f16_f128::f32_to_f16(fx, res)
} else {
res
};
CValue::by_val(res, in_lhs.layout())
}
pub(crate) fn codegen_ptr_binop<'tcx>(
fn codegen_ptr_binop<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
bin_op: BinOp,
in_lhs: CValue<'tcx>,
@ -457,15 +503,19 @@ pub(crate) fn codegen_ptr_binop<'tcx>(
// and `a.is_nan() ? b : (a <= b ? b : a)` for `maxnumf*`. NaN checks are done by comparing
// a float against itself. Only in case of NaN is it not equal to itself.
pub(crate) fn codegen_float_min(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
let a_ge_b = fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, a, b);
// FIXME(bytecodealliance/wasmtime#8312): Replace with Cranelift `fcmp` once
// `f16`/`f128` backend lowerings have been added to Cranelift.
let a_is_nan = codegen_f16_f128::fcmp(fx, FloatCC::NotEqual, a, a);
let a_ge_b = codegen_f16_f128::fcmp(fx, FloatCC::GreaterThanOrEqual, a, b);
let temp = fx.bcx.ins().select(a_ge_b, b, a);
fx.bcx.ins().select(a_is_nan, b, temp)
}
pub(crate) fn codegen_float_max(fx: &mut FunctionCx<'_, '_, '_>, a: Value, b: Value) -> Value {
let a_is_nan = fx.bcx.ins().fcmp(FloatCC::NotEqual, a, a);
let a_le_b = fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, a, b);
// FIXME(bytecodealliance/wasmtime#8312): Replace with Cranelift `fcmp` once
// `f16`/`f128` backend lowerings have been added to Cranelift.
let a_is_nan = codegen_f16_f128::fcmp(fx, FloatCC::NotEqual, a, a);
let a_le_b = codegen_f16_f128::fcmp(fx, FloatCC::LessThanOrEqual, a, b);
let temp = fx.bcx.ins().select(a_le_b, b, a);
fx.bcx.ins().select(a_is_nan, b, temp)
}

73
compiler/rustc_codegen_cranelift/src/pretty_clif.rs
View file

@ -8,40 +8,41 @@
//! target x86_64
//!
//! function u0:22(i64) -> i8, i8 system_v {
//! ; symbol _ZN97_$LT$example..IsNotEmpty$u20$as$u20$mini_core..FnOnce$LT$$LP$$RF$$RF$$u5b$u16$u5d$$C$$RP$$GT$$GT$9call_once17hd517c453d67c0915E
//! ; instance Instance { def: Item(WithOptConstParam { did: DefId(0:42 ~ example[4e51]::{impl#0}::call_once), const_param_did: None }), args: [ReErased, ReErased] }
//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, abi: Aggregate { sized: true }, fields: Arbitrary { offsets: [], memory_index: [] }, largest_niche: None, variants: Single { index: 0 } } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, abi: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), variants: Single { index: 0 } } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, abi: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, variants: Single { index: 0 } } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false }
//! ; symbol _ZN97_$LT$example..IsNotEmpty$u20$as$u20$mini_core..FnOnce$LT$$LP$$RF$$RF$$u5b$u16$u5d$$C$$RP$$GT$$GT$9call_once17hd361e9f5c3d1c4deE
//! ; instance Instance { def: Item(DefId(0:42 ~ example[3895]::{impl#0}::call_once)), args: ['{erased}, '{erased}] }
//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: Memory { sized: true }, fields: Arbitrary { offsets: [], memory_index: [] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 12266848898570219025 } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false }
//!
//! ; kind loc.idx param pass mode ty
//! ; ssa _0 (u8, u8) 2b 1, 8 var=(0, 1)
//! ; ssa _0 (u8, u8) 2b 1 var=(0, 1)
//! ; ret _0 - Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) (u8, u8)
//! ; arg _1 - Ignore IsNotEmpty
//! ; arg _2.0 = v0 Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) &&[u16]
//! ; arg _2.0 = v0 Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) &'{erased} &'{erased} [u16]
//!
//! ; kind local ty size align (abi,pref)
//! ; zst _1 IsNotEmpty 0b 1, 8 align=8,offset=
//! ; stack _2 (&&[u16],) 8b 8, 8 storage=ss0
//! ; ssa _3 &mut IsNotEmpty 8b 8, 8 var=2
//! ; kind local ty size align (abi)
//! ; zst _1 IsNotEmpty 0b 1 align=1,offset=
//! ; stack _2 (&'{erased} &'{erased} [u16],) 8b 8 storage=ss0
//! ; ssa _3 &'{erased} mut IsNotEmpty 8b 8 var=2
//!
//! ss0 = explicit_slot 16
//! ss0 = explicit_slot 16, align = 16
//! sig0 = (i64, i64) -> i8, i8 system_v
//! fn0 = colocated u0:23 sig0 ; Instance { def: Item(WithOptConstParam { did: DefId(0:46 ~ example[4e51]::{impl#1}::call_mut), const_param_did: None }), args: [ReErased, ReErased] }
//! fn0 = colocated u0:23 sig0 ; Instance { def: Item(DefId(0:46 ~ example[3895]::{impl#1}::call_mut)), args: ['{erased}, '{erased}] }
//!
//! block0(v0: i64):
//! nop
//! ; write_cvalue: Addr(Pointer { base: Stack(ss0), offset: Offset32(0) }, None): &&[u16] <- ByVal(v0): &&[u16]
//! ; write_cvalue: Addr(Pointer { base: Stack(ss0), offset: Offset32(0) }, None): &'{erased} &'{erased} [u16] <- ByVal(v0): &'{erased} &'{erased} [u16]
//! stack_store v0, ss0
//! jump block1
//!
//! block1:
//! nop
//! ; _3 = &mut _1
//! v1 = iconst.i64 8
//! ; write_cvalue: Var(_3, var2): &mut IsNotEmpty <- ByVal(v1): &mut IsNotEmpty
//! v1 = iconst.i64 1
//! ; write_cvalue: Var(_3, var2): &'{erased} mut IsNotEmpty <- ByVal(v1): &'{erased} mut IsNotEmpty
//! ;
//! ; _0 = <IsNotEmpty as mini_core::FnMut<(&&[u16],)>>::call_mut(move _3, _2)
//! ; _0 = <IsNotEmpty as mini_core::FnMut<(&&[u16],)>>::call_mut(move _3, copy _2)
//! v2 = stack_load.i64 ss0
//! v3, v4 = call fn0(v1, v2) ; v1 = 8
//! ; abi: FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: &mut IsNotEmpty, layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(1 bytes)) }) }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, backend_repr: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(8 bytes), randomization_seed: 281492156579847 } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, backend_repr: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, uninhabited: false, variants: Single { index: 0 }, max_repr_align: None, unadjusted_abi_align: Align(1 bytes), randomization_seed: 71776127651151873 } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false }
//! v3, v4 = call fn0(v1, v2) ; v1 = 1
//! v5 -> v3
//! v6 -> v4
//! ; write_cvalue: VarPair(_0, var0, var1): (u8, u8) <- ByValPair(v3, v4): (u8, u8)
@ -73,6 +74,7 @@ pub(crate) struct CommentWriter {
enabled: bool,
global_comments: Vec<String>,
entity_comments: FxHashMap<AnyEntity, String>,
inst_post_comments: FxHashMap<Inst, String>,
}
impl CommentWriter {
@ -95,7 +97,12 @@ impl CommentWriter {
vec![]
};
CommentWriter { enabled, global_comments, entity_comments: FxHashMap::default() }
CommentWriter {
enabled,
global_comments,
entity_comments: FxHashMap::default(),
inst_post_comments: FxHashMap::default(),
}
}
}
@ -127,6 +134,25 @@ impl CommentWriter {
}
}
}
pub(crate) fn add_post_comment<S: Into<String> + AsRef<str>>(
&mut self,
entity: Inst,
comment: S,
) {
debug_assert!(self.enabled);
use std::collections::hash_map::Entry;
match self.inst_post_comments.entry(entity) {
Entry::Occupied(mut occ) => {
occ.get_mut().push('\n');
occ.get_mut().push_str(comment.as_ref());
}
Entry::Vacant(vac) => {
vac.insert(comment.into());
}
}
}
}
impl FuncWriter for &'_ CommentWriter {
@ -188,10 +214,13 @@ impl FuncWriter for &'_ CommentWriter {
inst: Inst,
indent: usize,
) -> fmt::Result {
PlainWriter.write_instruction(w, func, aliases, inst, indent)?;
if let Some(comment) = self.entity_comments.get(&inst.into()) {
writeln!(w, "; {}", comment.replace('\n', "\n; "))?;
}
PlainWriter.write_instruction(w, func, aliases, inst, indent)?;
if let Some(comment) = self.inst_post_comments.get(&inst) {
writeln!(w, "; {}", comment.replace('\n', "\n; "))?;
}
Ok(())
}
}
@ -208,6 +237,14 @@ impl FunctionCx<'_, '_, '_> {
) {
self.clif_comments.add_comment(entity, comment);
}
pub(crate) fn add_post_comment<S: Into<String> + AsRef<str>>(
&mut self,
entity: Inst,
comment: S,
) {
self.clif_comments.add_post_comment(entity, comment);
}
}
pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {

38
compiler/rustc_codegen_cranelift/src/trap.rs
View file

@ -1,38 +0,0 @@
//! Helpers used to print a message and abort in case of certain panics and some detected UB.
use crate::prelude::*;
fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
let puts = fx
.module
.declare_function(
"puts",
Linkage::Import,
&Signature {
call_conv: fx.target_config.default_call_conv,
params: vec![AbiParam::new(fx.pointer_type)],
returns: vec![AbiParam::new(types::I32)],
},
)
.unwrap();
let puts = fx.module.declare_func_in_func(puts, &mut fx.bcx.func);
if fx.clif_comments.enabled() {
fx.add_comment(puts, "puts");
}
let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, fx.symbol_name, msg);
let msg_ptr = fx.anonymous_str(&real_msg);
fx.bcx.ins().call(puts, &[msg_ptr]);
}
/// Use this when something is unimplemented, but `libcore` or `libstd` requires it to codegen.
///
/// Trap code: user65535
pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
codegen_print(fx, msg.as_ref());
let one = fx.bcx.ins().iconst(types::I32, 1);
fx.lib_call("exit", vec![AbiParam::new(types::I32)], vec![], &[one]);
fx.bcx.ins().trap(TrapCode::user(3).unwrap());
}

24
compiler/rustc_codegen_cranelift/src/unwind_module.rs
View file

@ -1,10 +1,10 @@
use cranelift_codegen::Context;
use cranelift_codegen::control::ControlPlane;
use cranelift_codegen::ir::{Function, Signature};
use cranelift_codegen::ir::Signature;
use cranelift_codegen::isa::{TargetFrontendConfig, TargetIsa};
use cranelift_codegen::{Context, FinalizedMachReloc};
use cranelift_module::{
DataDescription, DataId, FuncId, FuncOrDataId, Linkage, Module, ModuleDeclarations,
ModuleResult,
ModuleReloc, ModuleResult,
};
use cranelift_object::{ObjectModule, ObjectProduct};
@ -17,8 +17,8 @@ pub(crate) struct UnwindModule<T> {
}
impl<T: Module> UnwindModule<T> {
pub(crate) fn new(module: T, pic_eh_frame: bool) -> Self {
let unwind_context = UnwindContext::new(module.isa(), pic_eh_frame);
pub(crate) fn new(mut module: T, pic_eh_frame: bool) -> Self {
let unwind_context = UnwindContext::new(&mut module, pic_eh_frame);
UnwindModule { module, unwind_context }
}
}
@ -33,13 +33,10 @@ impl UnwindModule<ObjectModule> {
#[cfg(feature = "jit")]
impl UnwindModule<cranelift_jit::JITModule> {
pub(crate) fn finalize_definitions(&mut self) {
pub(crate) fn finalize_definitions(mut self) -> cranelift_jit::JITModule {
self.module.finalize_definitions().unwrap();
let prev_unwind_context = std::mem::replace(
&mut self.unwind_context,
UnwindContext::new(self.module.isa(), false),
);
unsafe { prev_unwind_context.register_jit(&self.module) };
unsafe { self.unwind_context.register_jit(&self.module) };
self.module
}
}
@ -94,17 +91,16 @@ impl<T: Module> Module for UnwindModule<T> {
ctrl_plane: &mut ControlPlane,
) -> ModuleResult<()> {
self.module.define_function_with_control_plane(func, ctx, ctrl_plane)?;
self.unwind_context.add_function(func, ctx, self.module.isa());
self.unwind_context.add_function(&mut self.module, func, ctx);
Ok(())
}
fn define_function_bytes(
&mut self,
_func_id: FuncId,
_func: &Function,
_alignment: u64,
_bytes: &[u8],
_relocs: &[FinalizedMachReloc],
_relocs: &[ModuleReloc],
) -> ModuleResult<()> {
unimplemented!()
}

76
compiler/rustc_codegen_cranelift/src/value_and_place.rs
View file

@ -309,6 +309,7 @@ impl<'tcx> CValue<'tcx> {
match self.0 {
CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => unreachable!(),
CValueInner::ByRef(ptr, None) => {
let lane_idx = clif_intcast(fx, lane_idx, fx.pointer_type, false);
let field_offset = fx.bcx.ins().imul_imm(lane_idx, lane_layout.size.bytes() as i64);
let field_ptr = ptr.offset_value(fx, field_offset);
CValue::by_ref(field_ptr, lane_layout)
@ -324,7 +325,7 @@ impl<'tcx> CValue<'tcx> {
const_val: ty::ScalarInt,
) -> CValue<'tcx> {
assert_eq!(const_val.size(), layout.size, "{:#?}: {:?}", const_val, layout);
use cranelift_codegen::ir::immediates::{Ieee32, Ieee64};
use cranelift_codegen::ir::immediates::{Ieee16, Ieee32, Ieee64, Ieee128};
let clif_ty = fx.clif_type(layout.ty).unwrap();
@ -345,12 +346,24 @@ impl<'tcx> CValue<'tcx> {
let raw_val = const_val.size().truncate(const_val.to_bits(layout.size));
fx.bcx.ins().iconst(clif_ty, raw_val as i64)
}
ty::Float(FloatTy::F16) => {
fx.bcx.ins().f16const(Ieee16::with_bits(u16::try_from(const_val).unwrap()))
}
ty::Float(FloatTy::F32) => {
fx.bcx.ins().f32const(Ieee32::with_bits(u32::try_from(const_val).unwrap()))
}
ty::Float(FloatTy::F64) => {
fx.bcx.ins().f64const(Ieee64::with_bits(u64::try_from(const_val).unwrap()))
}
ty::Float(FloatTy::F128) => {
let value = fx
.bcx
.func
.dfg
.constants
.insert(Ieee128::with_bits(u128::try_from(const_val).unwrap()).into());
fx.bcx.ins().f128const(value)
}
_ => panic!(
"CValue::const_val for non bool/char/float/integer/pointer type {:?} is not allowed",
layout.ty
@ -563,27 +576,7 @@ impl<'tcx> CPlace<'tcx> {
src_ty,
dst_ty,
);
let data = match (src_ty, dst_ty) {
(_, _) if src_ty == dst_ty => data,
// This is a `write_cvalue_transmute`.
(types::I32, types::F32)
| (types::F32, types::I32)
| (types::I64, types::F64)
| (types::F64, types::I64) => codegen_bitcast(fx, dst_ty, data),
_ if src_ty.is_vector() && dst_ty.is_vector() => codegen_bitcast(fx, dst_ty, data),
_ if src_ty.is_vector() || dst_ty.is_vector() => {
// FIXME(bytecodealliance/wasmtime#6104) do something more efficient for transmutes between vectors and integers.
let ptr = fx.create_stack_slot(src_ty.bytes(), src_ty.bytes());
ptr.store(fx, data, MemFlags::trusted());
ptr.load(fx, dst_ty, MemFlags::trusted())
}
// `CValue`s should never contain SSA-only types, so if you ended
// up here having seen an error like `B1 -> I8`, then before
// calling `write_cvalue` you need to add a `bint` instruction.
_ => unreachable!("write_cvalue_transmute: {:?} -> {:?}", src_ty, dst_ty),
};
let data = if src_ty == dst_ty { data } else { codegen_bitcast(fx, dst_ty, data) };
//fx.bcx.set_val_label(data, cranelift_codegen::ir::ValueLabel::new(var.index()));
fx.bcx.def_var(var, data);
}
@ -591,13 +584,9 @@ impl<'tcx> CPlace<'tcx> {
assert_eq!(self.layout().size, from.layout().size);
if fx.clif_comments.enabled() {
use cranelift_codegen::cursor::{Cursor, CursorPosition};
let cur_block = match fx.bcx.cursor().position() {
CursorPosition::After(block) => block,
_ => unreachable!(),
};
fx.add_comment(
fx.bcx.func.layout.last_inst(cur_block).unwrap(),
let inst = fx.bcx.func.layout.last_inst(fx.bcx.current_block().unwrap()).unwrap();
fx.add_post_comment(
inst,
format!(
"{}: {:?}: {:?} <- {:?}: {:?}",
method,
@ -806,6 +795,35 @@ impl<'tcx> CPlace<'tcx> {
}
}
/// Write a value to an individual lane in a SIMD vector.
pub(crate) fn write_lane_dyn(
self,
fx: &mut FunctionCx<'_, '_, 'tcx>,
lane_idx: Value,
value: CValue<'tcx>,
) {
let layout = self.layout();
assert!(layout.ty.is_simd());
let (_lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
let lane_layout = fx.layout_of(lane_ty);
assert_eq!(lane_layout, value.layout());
match self.inner {
CPlaceInner::Var(_, _) => unreachable!(),
CPlaceInner::VarPair(_, _, _) => unreachable!(),
CPlaceInner::Addr(ptr, None) => {
let lane_idx = clif_intcast(fx, lane_idx, fx.pointer_type, false);
let field_offset = fx
.bcx
.ins()
.imul_imm(lane_idx, i64::try_from(lane_layout.size.bytes()).unwrap());
let field_ptr = ptr.offset_value(fx, field_offset);
CPlace::for_ptr(field_ptr, lane_layout).write_cvalue(fx, value);
}
CPlaceInner::Addr(_, Some(_)) => unreachable!(),
}
}
pub(crate) fn place_index(
self,
fx: &mut FunctionCx<'_, '_, 'tcx>,

7
compiler/rustc_codegen_cranelift/triagebot.toml Normal file
View file

@ -0,0 +1,7 @@
# Documentation at https://forge.rust-lang.org/triagebot/index.html
# Prevents un-canonicalized issue links (to avoid wrong issues being linked in r-l/rust)
[issue-links]
# Prevents mentions in commits to avoid users being spammed
[no-mentions]

48
compiler/rustc_codegen_ssa/src/mir/block.rs
View file

@ -1,6 +1,6 @@
use std::cmp;
use rustc_abi::{BackendRepr, ExternAbi, HasDataLayout, Reg, WrappingRange};
use rustc_abi::{BackendRepr, ExternAbi, HasDataLayout, Reg, Size, WrappingRange};
use rustc_ast as ast;
use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_data_structures::packed::Pu128;
@ -158,7 +158,7 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
llargs: &[Bx::Value],
destination: Option<(ReturnDest<'tcx, Bx::Value>, mir::BasicBlock)>,
mut unwind: mir::UnwindAction,
copied_constant_arguments: &[PlaceRef<'tcx, <Bx as BackendTypes>::Value>],
lifetime_ends_after_call: &[(Bx::Value, Size)],
instance: Option<Instance<'tcx>>,
mergeable_succ: bool,
) -> MergingSucc {
@ -245,8 +245,8 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
if let Some((ret_dest, target)) = destination {
bx.switch_to_block(fx.llbb(target));
fx.set_debug_loc(bx, self.terminator.source_info);
for tmp in copied_constant_arguments {
bx.lifetime_end(tmp.val.llval, tmp.layout.size);
for &(tmp, size) in lifetime_ends_after_call {
bx.lifetime_end(tmp, size);
}
fx.store_return(bx, ret_dest, &fn_abi.ret, invokeret);
}
@ -259,8 +259,8 @@ impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
}
if let Some((ret_dest, target)) = destination {
for tmp in copied_constant_arguments {
bx.lifetime_end(tmp.val.llval, tmp.layout.size);
for &(tmp, size) in lifetime_ends_after_call {
bx.lifetime_end(tmp, size);
}
fx.store_return(bx, ret_dest, &fn_abi.ret, llret);
self.funclet_br(fx, bx, target, mergeable_succ)
@ -1048,7 +1048,10 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
(args, None)
};
let mut copied_constant_arguments = vec![];
// When generating arguments we sometimes introduce temporary allocations with lifetime
// that extend for the duration of a call. Keep track of those allocations and their sizes
// to generate `lifetime_end` when the call returns.
let mut lifetime_ends_after_call: Vec<(Bx::Value, Size)> = Vec::new();
'make_args: for (i, arg) in first_args.iter().enumerate() {
let mut op = self.codegen_operand(bx, &arg.node);
@ -1136,12 +1139,18 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
bx.lifetime_start(tmp.val.llval, tmp.layout.size);
op.val.store(bx, tmp);
op.val = Ref(tmp.val);
copied_constant_arguments.push(tmp);
lifetime_ends_after_call.push((tmp.val.llval, tmp.layout.size));
}
_ => {}
}
self.codegen_argument(bx, op, &mut llargs, &fn_abi.args[i]);
self.codegen_argument(
bx,
op,
&mut llargs,
&fn_abi.args[i],
&mut lifetime_ends_after_call,
);
}
let num_untupled = untuple.map(|tup| {
self.codegen_arguments_untupled(
@ -1149,6 +1158,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
&tup.node,
&mut llargs,
&fn_abi.args[first_args.len()..],
&mut lifetime_ends_after_call,
)
});
@ -1173,7 +1183,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
);
let last_arg = fn_abi.args.last().unwrap();
self.codegen_argument(bx, location, &mut llargs, last_arg);
self.codegen_argument(
bx,
location,
&mut llargs,
last_arg,
&mut lifetime_ends_after_call,
);
}
let fn_ptr = match (instance, llfn) {
@ -1190,7 +1206,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
&llargs,
destination,
unwind,
&copied_constant_arguments,
&lifetime_ends_after_call,
instance,
mergeable_succ,
)
@ -1480,6 +1496,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
op: OperandRef<'tcx, Bx::Value>,
llargs: &mut Vec<Bx::Value>,
arg: &ArgAbi<'tcx, Ty<'tcx>>,
lifetime_ends_after_call: &mut Vec<(Bx::Value, Size)>,
) {
match arg.mode {
PassMode::Ignore => return,
@ -1518,7 +1535,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
None => arg.layout.align.abi,
};
let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
bx.lifetime_start(scratch.llval, arg.layout.size);
op.val.store(bx, scratch.with_type(arg.layout));
lifetime_ends_after_call.push((scratch.llval, arg.layout.size));
(scratch.llval, scratch.align, true)
}
PassMode::Cast { .. } => {
@ -1539,7 +1558,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
// alignment requirements may be higher than the type's alignment, so copy
// to a higher-aligned alloca.
let scratch = PlaceValue::alloca(bx, arg.layout.size, required_align);
bx.lifetime_start(scratch.llval, arg.layout.size);
bx.typed_place_copy(scratch, op_place_val, op.layout);
lifetime_ends_after_call.push((scratch.llval, arg.layout.size));
(scratch.llval, scratch.align, true)
} else {
(op_place_val.llval, op_place_val.align, true)
@ -1621,6 +1642,7 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
operand: &mir::Operand<'tcx>,
llargs: &mut Vec<Bx::Value>,
args: &[ArgAbi<'tcx, Ty<'tcx>>],
lifetime_ends_after_call: &mut Vec<(Bx::Value, Size)>,
) -> usize {
let tuple = self.codegen_operand(bx, operand);
@ -1633,13 +1655,13 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
for i in 0..tuple.layout.fields.count() {
let field_ptr = tuple_ptr.project_field(bx, i);
let field = bx.load_operand(field_ptr);
self.codegen_argument(bx, field, llargs, &args[i]);
self.codegen_argument(bx, field, llargs, &args[i], lifetime_ends_after_call);
}
} else {
// If the tuple is immediate, the elements are as well.
for i in 0..tuple.layout.fields.count() {
let op = tuple.extract_field(self, bx, i);
self.codegen_argument(bx, op, llargs, &args[i]);
self.codegen_argument(bx, op, llargs, &args[i], lifetime_ends_after_call);
}
}
tuple.layout.fields.count()

3
compiler/rustc_const_eval/messages.ftl
View file

@ -424,8 +424,7 @@ const_eval_unstable_in_stable_exposed =
.unstable_sugg = if the {$is_function_call2 ->
[true] caller
*[false] function
} is not (yet) meant to be exposed to stable, add `#[rustc_const_unstable]` (this is what you probably want to do)
.bypass_sugg = otherwise, as a last resort `#[rustc_allow_const_fn_unstable]` can be used to bypass stability checks (this requires team approval)
} is not (yet) meant to be exposed to stable const contexts, add `#[rustc_const_unstable]`
const_eval_unstable_intrinsic = `{$name}` is not yet stable as a const intrinsic
const_eval_unstable_intrinsic_suggestion = add `#![feature({$feature})]` to the crate attributes to enable

9
compiler/rustc_const_eval/src/const_eval/dummy_machine.rs
View file

@ -90,7 +90,7 @@ impl<'tcx> interpret::Machine<'tcx> for DummyMachine {
_instance: ty::Instance<'tcx>,
_abi: &FnAbi<'tcx, Ty<'tcx>>,
_args: &[interpret::FnArg<'tcx, Self::Provenance>],
_destination: &interpret::MPlaceTy<'tcx, Self::Provenance>,
_destination: &interpret::PlaceTy<'tcx, Self::Provenance>,
_target: Option<BasicBlock>,
_unwind: UnwindAction,
) -> interpret::InterpResult<'tcx, Option<(&'tcx Body<'tcx>, ty::Instance<'tcx>)>> {
@ -108,7 +108,7 @@ impl<'tcx> interpret::Machine<'tcx> for DummyMachine {
_ecx: &mut InterpCx<'tcx, Self>,
_instance: ty::Instance<'tcx>,
_args: &[interpret::OpTy<'tcx, Self::Provenance>],
_destination: &interpret::MPlaceTy<'tcx, Self::Provenance>,
_destination: &interpret::PlaceTy<'tcx, Self::Provenance>,
_target: Option<BasicBlock>,
_unwind: UnwindAction,
) -> interpret::InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
@ -197,4 +197,9 @@ impl<'tcx> interpret::Machine<'tcx> for DummyMachine {
) -> &'a mut Vec<interpret::Frame<'tcx, Self::Provenance, Self::FrameExtra>> {
unimplemented!()
}
fn get_default_alloc_params(
&self,
) -> <Self::Bytes as rustc_middle::mir::interpret::AllocBytes>::AllocParams {
}
}

7
compiler/rustc_const_eval/src/const_eval/eval_queries.rs
View file

@ -71,7 +71,12 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
// This can't use `init_stack_frame` since `body` is not a function,
// so computing its ABI would fail. It's also not worth it since there are no arguments to pass.
ecx.push_stack_frame_raw(cid.instance, body, &ret, StackPopCleanup::Root { cleanup: false })?;
ecx.push_stack_frame_raw(
cid.instance,
body,
&ret.clone().into(),
StackPopCleanup::Root { cleanup: false },
)?;
ecx.storage_live_for_always_live_locals()?;
// The main interpreter loop.

11
compiler/rustc_const_eval/src/const_eval/machine.rs
View file

@ -22,7 +22,7 @@ use crate::errors::{LongRunning, LongRunningWarn};
use crate::fluent_generated as fluent;
use crate::interpret::{
self, AllocId, AllocInit, AllocRange, ConstAllocation, CtfeProvenance, FnArg, Frame,
GlobalAlloc, ImmTy, InterpCx, InterpResult, MPlaceTy, OpTy, Pointer, RangeSet, Scalar,
GlobalAlloc, ImmTy, InterpCx, InterpResult, OpTy, PlaceTy, Pointer, RangeSet, Scalar,
compile_time_machine, interp_ok, throw_exhaust, throw_inval, throw_ub, throw_ub_custom,
throw_unsup, throw_unsup_format,
};
@ -226,7 +226,7 @@ impl<'tcx> CompileTimeInterpCx<'tcx> {
&mut self,
instance: ty::Instance<'tcx>,
args: &[FnArg<'tcx>],
_dest: &MPlaceTy<'tcx>,
_dest: &PlaceTy<'tcx>,
_ret: Option<mir::BasicBlock>,
) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
let def_id = instance.def_id();
@ -343,7 +343,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
orig_instance: ty::Instance<'tcx>,
_abi: &FnAbi<'tcx, Ty<'tcx>>,
args: &[FnArg<'tcx>],
dest: &MPlaceTy<'tcx>,
dest: &PlaceTy<'tcx>,
ret: Option<mir::BasicBlock>,
_unwind: mir::UnwindAction, // unwinding is not supported in consts
) -> InterpResult<'tcx, Option<(&'tcx mir::Body<'tcx>, ty::Instance<'tcx>)>> {
@ -385,7 +385,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
ecx: &mut InterpCx<'tcx, Self>,
instance: ty::Instance<'tcx>,
args: &[OpTy<'tcx>],
dest: &MPlaceTy<'tcx, Self::Provenance>,
dest: &PlaceTy<'tcx, Self::Provenance>,
target: Option<mir::BasicBlock>,
_unwind: mir::UnwindAction,
) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
@ -735,6 +735,9 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
Cow::Owned(compute_range())
}
}
fn get_default_alloc_params(&self) -> <Self::Bytes as mir::interpret::AllocBytes>::AllocParams {
}
}
// Please do not add any code below the above `Machine` trait impl. I (oli-obk) plan more cleanups

5
compiler/rustc_const_eval/src/errors.rs
View file

@ -58,11 +58,6 @@ pub(crate) struct UnstableInStableExposed {
code = "#[rustc_const_unstable(feature = \"...\", issue = \"...\")]\n",
applicability = "has-placeholders"
)]
#[suggestion(
const_eval_bypass_sugg,
code = "#[rustc_allow_const_fn_unstable({gate})]\n",
applicability = "has-placeholders"
)]
pub attr_span: Span,
}

52
compiler/rustc_const_eval/src/interpret/call.rs
View file

@ -339,7 +339,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
caller_fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
args: &[FnArg<'tcx, M::Provenance>],
with_caller_location: bool,
destination: &MPlaceTy<'tcx, M::Provenance>,
destination: &PlaceTy<'tcx, M::Provenance>,
mut stack_pop: StackPopCleanup,
) -> InterpResult<'tcx> {
// Compute callee information.
@ -487,7 +487,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
}
// Protect return place for in-place return value passing.
M::protect_in_place_function_argument(self, &destination)?;
// We only need to protect anything if this is actually an in-memory place.
if let Left(mplace) = destination.as_mplace_or_local() {
M::protect_in_place_function_argument(self, &mplace)?;
}
// Don't forget to mark "initially live" locals as live.
self.storage_live_for_always_live_locals()?;
@ -512,7 +515,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
(caller_abi, caller_fn_abi): (ExternAbi, &FnAbi<'tcx, Ty<'tcx>>),
args: &[FnArg<'tcx, M::Provenance>],
with_caller_location: bool,
destination: &MPlaceTy<'tcx, M::Provenance>,
destination: &PlaceTy<'tcx, M::Provenance>,
target: Option<mir::BasicBlock>,
unwind: mir::UnwindAction,
) -> InterpResult<'tcx> {
@ -776,10 +779,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// Note that we are using `pop_stack_frame_raw` and not `return_from_current_stack_frame`,
// as the latter "executes" the goto to the return block, but we don't want to,
// only the tail called function should return to the current return block.
M::before_stack_pop(self, self.frame())?;
let StackPopInfo { return_action, return_to_block, return_place } =
self.pop_stack_frame_raw(false)?;
let StackPopInfo { return_action, return_to_block, return_place } = self
.pop_stack_frame_raw(false, |_this, _return_place| {
// This function's return value is just discarded, the tail-callee will fill in the return place instead.
interp_ok(())
})?;
assert_eq!(return_action, ReturnAction::Normal);
@ -850,7 +854,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
(ExternAbi::Rust, fn_abi),
&[FnArg::Copy(arg.into())],
false,
&ret,
&ret.into(),
Some(target),
unwind,
)
@ -891,28 +895,16 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
throw_ub_custom!(fluent::const_eval_unwind_past_top);
}
M::before_stack_pop(self, self.frame())?;
// Copy return value. Must of course happen *before* we deallocate the locals.
// Must be *after* `before_stack_pop` as otherwise the return place might still be protected.
let copy_ret_result = if !unwinding {
let op = self
.local_to_op(mir::RETURN_PLACE, None)
.expect("return place should always be live");
let dest = self.frame().return_place.clone();
let res = self.copy_op_allow_transmute(&op, &dest);
trace!("return value: {:?}", self.dump_place(&dest.into()));
// We delay actually short-circuiting on this error until *after* the stack frame is
// popped, since we want this error to be attributed to the caller, whose type defines
// this transmute.
res
} else {
// Get out the return value. Must happen *before* the frame is popped as we have to get the
// local's value out.
let return_op =
self.local_to_op(mir::RETURN_PLACE, None).expect("return place should always be live");
// Do the actual pop + copy.
let stack_pop_info = self.pop_stack_frame_raw(unwinding, |this, return_place| {
this.copy_op_allow_transmute(&return_op, return_place)?;
trace!("return value: {:?}", this.dump_place(return_place));
interp_ok(())
};
// All right, now it is time to actually pop the frame.
// An error here takes precedence over the copy error.
let (stack_pop_info, ()) = self.pop_stack_frame_raw(unwinding).and(copy_ret_result)?;
})?;
match stack_pop_info.return_action {
ReturnAction::Normal => {}
@ -924,7 +916,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// If we are not doing cleanup, also skip everything else.
assert!(self.stack().is_empty(), "only the topmost frame should ever be leaked");
assert!(!unwinding, "tried to skip cleanup during unwinding");
// Skip machine hook.
// Don't jump anywhere.
return interp_ok(());
}
}

24
compiler/rustc_const_eval/src/interpret/intrinsics.rs
View file

@ -17,16 +17,16 @@ use tracing::trace;
use super::memory::MemoryKind;
use super::util::ensure_monomorphic_enough;
use super::{
Allocation, CheckInAllocMsg, ConstAllocation, GlobalId, ImmTy, InterpCx, InterpResult,
MPlaceTy, Machine, OpTy, Pointer, PointerArithmetic, Provenance, Scalar, err_inval,
err_ub_custom, err_unsup_format, interp_ok, throw_inval, throw_ub_custom, throw_ub_format,
Allocation, CheckInAllocMsg, ConstAllocation, GlobalId, ImmTy, InterpCx, InterpResult, Machine,
OpTy, PlaceTy, Pointer, PointerArithmetic, Provenance, Scalar, err_inval, err_ub_custom,
err_unsup_format, interp_ok, throw_inval, throw_ub_custom, throw_ub_format,
};
use crate::fluent_generated as fluent;
/// Directly returns an `Allocation` containing an absolute path representation of the given type.
pub(crate) fn alloc_type_name<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> ConstAllocation<'tcx> {
let path = crate::util::type_name(tcx, ty);
let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes());
let alloc = Allocation::from_bytes_byte_aligned_immutable(path.into_bytes(), ());
tcx.mk_const_alloc(alloc)
}
@ -112,7 +112,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
&mut self,
instance: ty::Instance<'tcx>,
args: &[OpTy<'tcx, M::Provenance>],
dest: &MPlaceTy<'tcx, M::Provenance>,
dest: &PlaceTy<'tcx, M::Provenance>,
ret: Option<mir::BasicBlock>,
) -> InterpResult<'tcx, bool> {
let instance_args = instance.args;
@ -587,7 +587,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
&mut self,
a: &ImmTy<'tcx, M::Provenance>,
b: &ImmTy<'tcx, M::Provenance>,
dest: &MPlaceTy<'tcx, M::Provenance>,
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx> {
assert_eq!(a.layout.ty, b.layout.ty);
assert_matches!(a.layout.ty.kind(), ty::Int(..) | ty::Uint(..));
@ -801,7 +801,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
fn float_min_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &MPlaceTy<'tcx, M::Provenance>,
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
@ -822,7 +822,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
fn float_max_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &MPlaceTy<'tcx, M::Provenance>,
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
@ -843,7 +843,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
fn float_minimum_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &MPlaceTy<'tcx, M::Provenance>,
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
@ -859,7 +859,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
fn float_maximum_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &MPlaceTy<'tcx, M::Provenance>,
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
@ -875,7 +875,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
fn float_copysign_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &MPlaceTy<'tcx, M::Provenance>,
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,
@ -890,7 +890,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
fn float_abs_intrinsic<F>(
&mut self,
args: &[OpTy<'tcx, M::Provenance>],
dest: &MPlaceTy<'tcx, M::Provenance>,
dest: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, ()>
where
F: rustc_apfloat::Float + rustc_apfloat::FloatConvert<F> + Into<Scalar<M::Provenance>>,

20
compiler/rustc_const_eval/src/interpret/machine.rs
View file

@ -208,7 +208,7 @@ pub trait Machine<'tcx>: Sized {
instance: ty::Instance<'tcx>,
abi: &FnAbi<'tcx, Ty<'tcx>>,
args: &[FnArg<'tcx, Self::Provenance>],
destination: &MPlaceTy<'tcx, Self::Provenance>,
destination: &PlaceTy<'tcx, Self::Provenance>,
target: Option<mir::BasicBlock>,
unwind: mir::UnwindAction,
) -> InterpResult<'tcx, Option<(&'tcx mir::Body<'tcx>, ty::Instance<'tcx>)>>;
@ -220,7 +220,7 @@ pub trait Machine<'tcx>: Sized {
fn_val: Self::ExtraFnVal,
abi: &FnAbi<'tcx, Ty<'tcx>>,
args: &[FnArg<'tcx, Self::Provenance>],
destination: &MPlaceTy<'tcx, Self::Provenance>,
destination: &PlaceTy<'tcx, Self::Provenance>,
target: Option<mir::BasicBlock>,
unwind: mir::UnwindAction,
) -> InterpResult<'tcx>;
@ -234,7 +234,7 @@ pub trait Machine<'tcx>: Sized {
ecx: &mut InterpCx<'tcx, Self>,
instance: ty::Instance<'tcx>,
args: &[OpTy<'tcx, Self::Provenance>],
destination: &MPlaceTy<'tcx, Self::Provenance>,
destination: &PlaceTy<'tcx, Self::Provenance>,
target: Option<mir::BasicBlock>,
unwind: mir::UnwindAction,
) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>>;
@ -536,11 +536,9 @@ pub trait Machine<'tcx>: Sized {
interp_ok(())
}
/// Called just before the return value is copied to the caller-provided return place.
fn before_stack_pop(
_ecx: &InterpCx<'tcx, Self>,
_frame: &Frame<'tcx, Self::Provenance, Self::FrameExtra>,
) -> InterpResult<'tcx> {
/// Called just before the frame is removed from the stack (followed by return value copy and
/// local cleanup).
fn before_stack_pop(_ecx: &mut InterpCx<'tcx, Self>) -> InterpResult<'tcx> {
interp_ok(())
}
@ -628,6 +626,10 @@ pub trait Machine<'tcx>: Sized {
// Default to no caching.
Cow::Owned(compute_range())
}
/// Compute the value passed to the constructors of the `AllocBytes` type for
/// abstract machine allocations.
fn get_default_alloc_params(&self) -> <Self::Bytes as AllocBytes>::AllocParams;
}
/// A lot of the flexibility above is just needed for `Miri`, but all "compile-time" machines
@ -675,7 +677,7 @@ pub macro compile_time_machine(<$tcx: lifetime>) {
fn_val: !,
_abi: &FnAbi<$tcx, Ty<$tcx>>,
_args: &[FnArg<$tcx>],
_destination: &MPlaceTy<$tcx, Self::Provenance>,
_destination: &PlaceTy<$tcx, Self::Provenance>,
_target: Option<mir::BasicBlock>,
_unwind: mir::UnwindAction,
) -> InterpResult<$tcx> {

8
compiler/rustc_const_eval/src/interpret/memory.rs
View file

@ -233,10 +233,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
kind: MemoryKind<M::MemoryKind>,
init: AllocInit,
) -> InterpResult<'tcx, Pointer<M::Provenance>> {
let params = self.machine.get_default_alloc_params();
let alloc = if M::PANIC_ON_ALLOC_FAIL {
Allocation::new(size, align, init)
Allocation::new(size, align, init, params)
} else {
Allocation::try_new(size, align, init)?
Allocation::try_new(size, align, init, params)?
};
self.insert_allocation(alloc, kind)
}
@ -248,7 +249,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
kind: MemoryKind<M::MemoryKind>,
mutability: Mutability,
) -> InterpResult<'tcx, Pointer<M::Provenance>> {
let alloc = Allocation::from_bytes(bytes, align, mutability);
let params = self.machine.get_default_alloc_params();
let alloc = Allocation::from_bytes(bytes, align, mutability, params);
self.insert_allocation(alloc, kind)
}

62
compiler/rustc_const_eval/src/interpret/stack.rs
View file

@ -15,9 +15,9 @@ use rustc_span::Span;
use tracing::{info_span, instrument, trace};
use super::{
AllocId, CtfeProvenance, Immediate, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace,
MemPlaceMeta, MemoryKind, Operand, Pointer, Provenance, ReturnAction, Scalar,
from_known_layout, interp_ok, throw_ub, throw_unsup,
AllocId, CtfeProvenance, Immediate, InterpCx, InterpResult, Machine, MemPlace, MemPlaceMeta,
MemoryKind, Operand, PlaceTy, Pointer, Provenance, ReturnAction, Scalar, from_known_layout,
interp_ok, throw_ub, throw_unsup,
};
use crate::errors;
@ -76,8 +76,10 @@ pub struct Frame<'tcx, Prov: Provenance = CtfeProvenance, Extra = ()> {
return_to_block: StackPopCleanup,
/// The location where the result of the current stack frame should be written to,
/// and its layout in the caller.
pub return_place: MPlaceTy<'tcx, Prov>,
/// and its layout in the caller. This place is to be interpreted relative to the
/// *caller's* stack frame. We use a `PlaceTy` instead of an `MPlaceTy` since this
/// avoids having to move *all* return places into Miri's memory.
pub return_place: PlaceTy<'tcx, Prov>,
/// The list of locals for this stack frame, stored in order as
/// `[return_ptr, arguments..., variables..., temporaries...]`.
@ -129,7 +131,7 @@ pub struct StackPopInfo<'tcx, Prov: Provenance> {
pub return_to_block: StackPopCleanup,
/// [`return_place`](Frame::return_place) of the popped stack frame.
pub return_place: MPlaceTy<'tcx, Prov>,
pub return_place: PlaceTy<'tcx, Prov>,
}
/// State of a local variable including a memoized layout
@ -353,7 +355,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
&mut self,
instance: ty::Instance<'tcx>,
body: &'tcx mir::Body<'tcx>,
return_place: &MPlaceTy<'tcx, M::Provenance>,
return_place: &PlaceTy<'tcx, M::Provenance>,
return_to_block: StackPopCleanup,
) -> InterpResult<'tcx> {
trace!("body: {:#?}", body);
@ -404,9 +406,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
/// it.
///
/// This also deallocates locals, if necessary.
/// `copy_ret_val` gets called after the frame has been taken from the stack but before the locals have been deallocated.
///
/// [`M::before_stack_pop`] should be called before calling this function.
/// [`M::after_stack_pop`] is called by this function automatically.
/// [`M::before_stack_pop`] and [`M::after_stack_pop`] are called by this function
/// automatically.
///
/// The high-level version of this is `return_from_current_stack_frame`.
///
@ -415,47 +418,44 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
pub(super) fn pop_stack_frame_raw(
&mut self,
unwinding: bool,
copy_ret_val: impl FnOnce(&mut Self, &PlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx>,
) -> InterpResult<'tcx, StackPopInfo<'tcx, M::Provenance>> {
let cleanup = self.cleanup_current_frame_locals()?;
M::before_stack_pop(self)?;
let frame =
self.stack_mut().pop().expect("tried to pop a stack frame, but there were none");
// Copy return value (unless we are unwinding).
if !unwinding {
copy_ret_val(self, &frame.return_place)?;
}
let return_to_block = frame.return_to_block;
let return_place = frame.return_place.clone();
let return_action;
if cleanup {
return_action = M::after_stack_pop(self, frame, unwinding)?;
assert_ne!(return_action, ReturnAction::NoCleanup);
} else {
return_action = ReturnAction::NoCleanup;
};
interp_ok(StackPopInfo { return_action, return_to_block, return_place })
}
/// A private helper for [`pop_stack_frame_raw`](InterpCx::pop_stack_frame_raw).
/// Returns `true` if cleanup has been done, `false` otherwise.
fn cleanup_current_frame_locals(&mut self) -> InterpResult<'tcx, bool> {
// Cleanup: deallocate locals.
// Usually we want to clean up (deallocate locals), but in a few rare cases we don't.
// We do this while the frame is still on the stack, so errors point to the callee.
let return_to_block = self.frame().return_to_block;
let cleanup = match return_to_block {
StackPopCleanup::Goto { .. } => true,
StackPopCleanup::Root { cleanup, .. } => cleanup,
};
if cleanup {
let return_action = if cleanup {
// We need to take the locals out, since we need to mutate while iterating.
let locals = mem::take(&mut self.frame_mut().locals);
for local in &locals {
for local in &frame.locals {
self.deallocate_local(local.value)?;
}
}
interp_ok(cleanup)
// Call the machine hook, which determines the next steps.
let return_action = M::after_stack_pop(self, frame, unwinding)?;
assert_ne!(return_action, ReturnAction::NoCleanup);
return_action
} else {
// We also skip the machine hook when there's no cleanup. This not a real "pop" anyway.
ReturnAction::NoCleanup
};
interp_ok(StackPopInfo { return_action, return_to_block, return_place })
}
/// In the current stack frame, mark all locals as live that are not arguments and don't have

2
compiler/rustc_const_eval/src/interpret/step.rs
View file

@ -506,7 +506,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location } =
self.eval_callee_and_args(terminator, func, args)?;
let destination = self.force_allocation(&self.eval_place(destination)?)?;
let destination = self.eval_place(destination)?;
self.init_fn_call(
callee,
(fn_sig.abi, fn_abi),

2
compiler/rustc_const_eval/src/interpret/util.rs
View file

@ -38,7 +38,7 @@ pub(crate) fn create_static_alloc<'tcx>(
static_def_id: LocalDefId,
layout: TyAndLayout<'tcx>,
) -> InterpResult<'tcx, MPlaceTy<'tcx>> {
let alloc = Allocation::try_new(layout.size, layout.align.abi, AllocInit::Uninit)?;
let alloc = Allocation::try_new(layout.size, layout.align.abi, AllocInit::Uninit, ())?;
let alloc_id = ecx.tcx.reserve_and_set_static_alloc(static_def_id.into());
assert_eq!(ecx.machine.static_root_ids, None);
ecx.machine.static_root_ids = Some((alloc_id, static_def_id));

2
compiler/rustc_hir_analysis/src/check/wfcheck.rs
View file

@ -742,7 +742,7 @@ fn ty_known_to_outlive<'tcx>(
region: ty::Region<'tcx>,
) -> bool {
test_region_obligations(tcx, id, param_env, wf_tys, |infcx| {
infcx.register_region_obligation(infer::RegionObligation {
infcx.register_type_outlives_constraint_inner(infer::TypeOutlivesConstraint {
sub_region: region,
sup_type: ty,
origin: infer::RelateParamBound(DUMMY_SP, ty, None),

2
compiler/rustc_hir_typeck/messages.ftl
View file

@ -169,7 +169,7 @@ hir_typeck_option_result_copied = use `{$def_path}::copied` to copy the value in
hir_typeck_pass_to_variadic_function = can't pass `{$ty}` to variadic function
.suggestion = cast the value to `{$cast_ty}`
.teach_help = certain types, like `{$ty}`, must be casted before passing them to a variadic function, because of arcane ABI rules dictated by the C standard
.teach_help = certain types, like `{$ty}`, must be cast before passing them to a variadic function to match the implicit cast that a C compiler would perform as part of C's numeric promotion rules
hir_typeck_ptr_cast_add_auto_to_object = cannot add {$traits_len ->
[1] auto trait {$traits}

18
compiler/rustc_hir_typeck/src/coercion.rs
View file

@ -47,7 +47,7 @@ use rustc_infer::infer::relate::RelateResult;
use rustc_infer::infer::{Coercion, DefineOpaqueTypes, InferOk, InferResult};
use rustc_infer::traits::{
IfExpressionCause, MatchExpressionArmCause, Obligation, PredicateObligation,
PredicateObligations,
PredicateObligations, SelectionError,
};
use rustc_middle::span_bug;
use rustc_middle::ty::adjustment::{
@ -677,7 +677,21 @@ impl<'f, 'tcx> Coerce<'f, 'tcx> {
return Err(TypeError::Mismatch);
}
// Dyn-compatibility violations or miscellaneous.
Err(SelectionError::TraitDynIncompatible(_)) => {
// Dyn compatibility errors in coercion will *always* be due to the
// fact that the RHS of the coercion is a non-dyn compatible `dyn Trait`
// writen in source somewhere (otherwise we will never have lowered
// the dyn trait from HIR to middle).
//
// There's no reason to emit yet another dyn compatibility error,
// especially since the span will differ slightly and thus not be
// deduplicated at all!
self.fcx.set_tainted_by_errors(
self.fcx
.dcx()
.span_delayed_bug(self.cause.span, "dyn compatibility during coercion"),
);
}
Err(err) => {
let guar = self.err_ctxt().report_selection_error(
obligation.clone(),

16
compiler/rustc_hir_typeck/src/expr_use_visitor.rs
View file

@ -13,6 +13,7 @@ use hir::Expr;
use hir::def::DefKind;
use hir::pat_util::EnumerateAndAdjustIterator as _;
use rustc_abi::{FIRST_VARIANT, FieldIdx, VariantIdx};
use rustc_ast::UnsafeBinderCastKind;
use rustc_data_structures::fx::FxIndexMap;
use rustc_hir::def::{CtorOf, Res};
use rustc_hir::def_id::LocalDefId;
@ -1393,10 +1394,18 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx
self.cat_res(expr.hir_id, expr.span, expr_ty, res)
}
// both type ascription and unsafe binder casts don't affect
// the place-ness of the subexpression.
// type ascription doesn't affect the place-ness of the subexpression.
hir::ExprKind::Type(e, _) => self.cat_expr(e),
hir::ExprKind::UnsafeBinderCast(_, e, _) => self.cat_expr(e),
hir::ExprKind::UnsafeBinderCast(UnsafeBinderCastKind::Unwrap, e, _) => {
let base = self.cat_expr(e)?;
Ok(self.cat_projection(
expr.hir_id,
base,
expr_ty,
ProjectionKind::UnwrapUnsafeBinder,
))
}
hir::ExprKind::AddrOf(..)
| hir::ExprKind::Call(..)
@ -1427,6 +1436,7 @@ impl<'tcx, Cx: TypeInformationCtxt<'tcx>, D: Delegate<'tcx>> ExprUseVisitor<'tcx
| hir::ExprKind::Repeat(..)
| hir::ExprKind::InlineAsm(..)
| hir::ExprKind::OffsetOf(..)
| hir::ExprKind::UnsafeBinderCast(UnsafeBinderCastKind::Wrap, ..)
| hir::ExprKind::Err(_) => Ok(self.cat_rvalue(expr.hir_id, expr_ty)),
}
}

6
compiler/rustc_hir_typeck/src/method/suggest.rs
View file

@ -599,7 +599,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
let tcx = self.tcx;
let rcvr_ty = self.resolve_vars_if_possible(rcvr_ty);
let mut ty_file = None;
let (mut ty_str, short_ty_str) =
let (ty_str, short_ty_str) =
if trait_missing_method && let ty::Dynamic(predicates, _, _) = rcvr_ty.kind() {
(predicates.to_string(), with_forced_trimmed_paths!(predicates.to_string()))
} else {
@ -738,10 +738,6 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
err.span_label(within_macro_span, "due to this macro variable");
}
if short_ty_str.len() < ty_str.len() && ty_str.len() > 10 {
ty_str = short_ty_str;
}
if rcvr_ty.references_error() {
err.downgrade_to_delayed_bug();
}

8
compiler/rustc_hir_typeck/src/upvar.rs
View file

@ -902,7 +902,9 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
fn is_field<'a>(p: &&Projection<'a>) -> bool {
match p.kind {
ProjectionKind::Field(_, _) => true,
ProjectionKind::Deref | ProjectionKind::OpaqueCast => false,
ProjectionKind::Deref
| ProjectionKind::OpaqueCast
| ProjectionKind::UnwrapUnsafeBinder => false,
p @ (ProjectionKind::Subslice | ProjectionKind::Index) => {
bug!("ProjectionKind {:?} was unexpected", p)
}
@ -2197,7 +2199,8 @@ fn restrict_capture_precision(
}
ProjectionKind::Deref => {}
ProjectionKind::OpaqueCast => {}
ProjectionKind::Field(..) => {} // ignore
ProjectionKind::Field(..) => {}
ProjectionKind::UnwrapUnsafeBinder => {}
}
}
@ -2268,6 +2271,7 @@ fn construct_place_string<'tcx>(tcx: TyCtxt<'_>, place: &Place<'tcx>) -> String
ProjectionKind::Index => String::from("Index"),
ProjectionKind::Subslice => String::from("Subslice"),
ProjectionKind::OpaqueCast => String::from("OpaqueCast"),
ProjectionKind::UnwrapUnsafeBinder => String::from("UnwrapUnsafeBinder"),
};
if i != 0 {
projections_str.push(',');

4
compiler/rustc_infer/src/infer/canonical/canonicalizer.rs
View file

@ -493,6 +493,10 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> {
ct
}
}
fn fold_predicate(&mut self, p: ty::Predicate<'tcx>) -> ty::Predicate<'tcx> {
if p.flags().intersects(self.needs_canonical_flags) { p.super_fold_with(self) } else { p }
}
}
impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> {

72
compiler/rustc_infer/src/infer/canonical/query_response.rs
View file

@ -12,23 +12,20 @@ use std::iter;
use rustc_index::{Idx, IndexVec};
use rustc_middle::arena::ArenaAllocatable;
use rustc_middle::bug;
use rustc_middle::mir::ConstraintCategory;
use rustc_middle::ty::{self, BoundVar, GenericArg, GenericArgKind, Ty, TyCtxt, TypeFoldable};
use rustc_middle::{bug, span_bug};
use tracing::{debug, instrument};
use crate::infer::canonical::instantiate::{CanonicalExt, instantiate_value};
use crate::infer::canonical::{
Canonical, CanonicalQueryResponse, CanonicalVarValues, Certainty, OriginalQueryValues,
QueryOutlivesConstraint, QueryRegionConstraints, QueryResponse,
QueryRegionConstraints, QueryResponse,
};
use crate::infer::region_constraints::{Constraint, RegionConstraintData};
use crate::infer::{DefineOpaqueTypes, InferCtxt, InferOk, InferResult};
use crate::infer::{DefineOpaqueTypes, InferCtxt, InferOk, InferResult, SubregionOrigin};
use crate::traits::query::NoSolution;
use crate::traits::{
Obligation, ObligationCause, PredicateObligation, PredicateObligations, ScrubbedTraitError,
TraitEngine,
};
use crate::traits::{ObligationCause, PredicateObligations, ScrubbedTraitError, TraitEngine};
impl<'tcx> InferCtxt<'tcx> {
/// This method is meant to be invoked as the final step of a canonical query
@ -169,15 +166,13 @@ impl<'tcx> InferCtxt<'tcx> {
where
R: Debug + TypeFoldable<TyCtxt<'tcx>>,
{
let InferOk { value: result_args, mut obligations } =
let InferOk { value: result_args, obligations } =
self.query_response_instantiation(cause, param_env, original_values, query_response)?;
obligations.extend(self.query_outlives_constraints_into_obligations(
cause,
param_env,
&query_response.value.region_constraints.outlives,
&result_args,
));
for (predicate, _category) in &query_response.value.region_constraints.outlives {
let predicate = instantiate_value(self.tcx, &result_args, *predicate);
self.register_outlives_constraint(predicate, cause);
}
let user_result: R =
query_response.instantiate_projected(self.tcx, &result_args, |q_r| q_r.value.clone());
@ -525,47 +520,6 @@ impl<'tcx> InferCtxt<'tcx> {
self.unify_canonical_vars(cause, param_env, original_values, instantiated_query_response)
}
/// Converts the region constraints resulting from a query into an
/// iterator of obligations.
fn query_outlives_constraints_into_obligations(
&self,
cause: &ObligationCause<'tcx>,
param_env: ty::ParamEnv<'tcx>,
uninstantiated_region_constraints: &[QueryOutlivesConstraint<'tcx>],
result_args: &CanonicalVarValues<'tcx>,
) -> impl Iterator<Item = PredicateObligation<'tcx>> {
uninstantiated_region_constraints.iter().map(move |&constraint| {
let predicate = instantiate_value(self.tcx, result_args, constraint);
self.query_outlives_constraint_to_obligation(predicate, cause.clone(), param_env)
})
}
pub fn query_outlives_constraint_to_obligation(
&self,
(predicate, _): QueryOutlivesConstraint<'tcx>,
cause: ObligationCause<'tcx>,
param_env: ty::ParamEnv<'tcx>,
) -> Obligation<'tcx, ty::Predicate<'tcx>> {
let ty::OutlivesPredicate(k1, r2) = predicate;
let atom = match k1.unpack() {
GenericArgKind::Lifetime(r1) => ty::PredicateKind::Clause(
ty::ClauseKind::RegionOutlives(ty::OutlivesPredicate(r1, r2)),
),
GenericArgKind::Type(t1) => ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(
ty::OutlivesPredicate(t1, r2),
)),
GenericArgKind::Const(..) => {
// Consts cannot outlive one another, so we don't expect to
// encounter this branch.
span_bug!(cause.span, "unexpected const outlives {:?}", predicate);
}
};
let predicate = ty::Binder::dummy(atom);
Obligation::new(self.tcx, cause, param_env, predicate)
}
/// Given two sets of values for the same set of canonical variables, unify them.
/// The second set is produced lazily by supplying indices from the first set.
fn unify_canonical_vars(
@ -593,10 +547,10 @@ impl<'tcx> InferCtxt<'tcx> {
// no action needed
}
(GenericArgKind::Lifetime(v1), GenericArgKind::Lifetime(v2)) => {
obligations.extend(
self.at(cause, param_env)
.eq(DefineOpaqueTypes::Yes, v1, v2)?
.into_obligations(),
self.inner.borrow_mut().unwrap_region_constraints().make_eqregion(
SubregionOrigin::RelateRegionParamBound(cause.span, None),
v1,
v2,
);
}
(GenericArgKind::Const(v1), GenericArgKind::Const(v2)) => {

53
compiler/rustc_infer/src/infer/context.rs
View file

@ -89,6 +89,57 @@ impl<'tcx> rustc_type_ir::InferCtxtLike for InferCtxt<'tcx> {
self.inner.borrow_mut().unwrap_region_constraints().opportunistic_resolve_var(self.tcx, vid)
}
fn is_changed_arg(&self, arg: ty::GenericArg<'tcx>) -> bool {
match arg.unpack() {
ty::GenericArgKind::Lifetime(_) => {
// Lifetimes should not change affect trait selection.
false
}
ty::GenericArgKind::Type(ty) => {
if let ty::Infer(infer_ty) = *ty.kind() {
match infer_ty {
ty::InferTy::TyVar(vid) => {
!self.probe_ty_var(vid).is_err_and(|_| self.root_var(vid) == vid)
}
ty::InferTy::IntVar(vid) => {
let mut inner = self.inner.borrow_mut();
!matches!(
inner.int_unification_table().probe_value(vid),
ty::IntVarValue::Unknown
if inner.int_unification_table().find(vid) == vid
)
}
ty::InferTy::FloatVar(vid) => {
let mut inner = self.inner.borrow_mut();
!matches!(
inner.float_unification_table().probe_value(vid),
ty::FloatVarValue::Unknown
if inner.float_unification_table().find(vid) == vid
)
}
ty::InferTy::FreshTy(_)
| ty::InferTy::FreshIntTy(_)
| ty::InferTy::FreshFloatTy(_) => true,
}
} else {
true
}
}
ty::GenericArgKind::Const(ct) => {
if let ty::ConstKind::Infer(infer_ct) = ct.kind() {
match infer_ct {
ty::InferConst::Var(vid) => !self
.probe_const_var(vid)
.is_err_and(|_| self.root_const_var(vid) == vid),
ty::InferConst::Fresh(_) => true,
}
} else {
true
}
}
}
}
fn next_region_infer(&self) -> ty::Region<'tcx> {
self.next_region_var(RegionVariableOrigin::MiscVariable(DUMMY_SP))
}
@ -214,7 +265,7 @@ impl<'tcx> rustc_type_ir::InferCtxtLike for InferCtxt<'tcx> {
}
fn register_ty_outlives(&self, ty: Ty<'tcx>, r: ty::Region<'tcx>, span: Span) {
self.register_region_obligation_with_cause(ty, r, &ObligationCause::dummy_with_span(span));
self.register_type_outlives_constraint(ty, r, &ObligationCause::dummy_with_span(span));
}
type OpaqueTypeStorageEntries = OpaqueTypeStorageEntries;

19
compiler/rustc_infer/src/infer/mod.rs
View file

@ -150,7 +150,7 @@ pub struct InferCtxtInner<'tcx> {
/// for each body-id in this map, which will process the
/// obligations within. This is expected to be done 'late enough'
/// that all type inference variables have been bound and so forth.
region_obligations: Vec<RegionObligation<'tcx>>,
region_obligations: Vec<TypeOutlivesConstraint<'tcx>>,
/// Caches for opaque type inference.
opaque_type_storage: OpaqueTypeStorage<'tcx>,
@ -173,7 +173,7 @@ impl<'tcx> InferCtxtInner<'tcx> {
}
#[inline]
pub fn region_obligations(&self) -> &[RegionObligation<'tcx>] {
pub fn region_obligations(&self) -> &[TypeOutlivesConstraint<'tcx>] {
&self.region_obligations
}
@ -488,7 +488,7 @@ impl fmt::Display for FixupError {
/// See the `region_obligations` field for more information.
#[derive(Clone, Debug)]
pub struct RegionObligation<'tcx> {
pub struct TypeOutlivesConstraint<'tcx> {
pub sub_region: ty::Region<'tcx>,
pub sup_type: Ty<'tcx>,
pub origin: SubregionOrigin<'tcx>,
@ -738,19 +738,6 @@ impl<'tcx> InferCtxt<'tcx> {
})
}
pub fn region_outlives_predicate(
&self,
cause: &traits::ObligationCause<'tcx>,
predicate: ty::PolyRegionOutlivesPredicate<'tcx>,
) {
self.enter_forall(predicate, |ty::OutlivesPredicate(r_a, r_b)| {
let origin = SubregionOrigin::from_obligation_cause(cause, || {
RelateRegionParamBound(cause.span, None)
});
self.sub_regions(origin, r_b, r_a); // `b : a` ==> `a <= b`
})
}
/// Number of type variables created so far.
pub fn num_ty_vars(&self) -> usize {
self.inner.borrow_mut().type_variables().num_vars()

6
compiler/rustc_infer/src/infer/opaque_types/table.rs
View file

@ -24,6 +24,12 @@ pub struct OpaqueTypeStorageEntries {
duplicate_entries: usize,
}
impl rustc_type_ir::inherent::OpaqueTypeStorageEntries for OpaqueTypeStorageEntries {
fn needs_reevaluation(self, canonicalized: usize) -> bool {
self.opaque_types != canonicalized
}
}
impl<'tcx> OpaqueTypeStorage<'tcx> {
#[instrument(level = "debug")]
pub(crate) fn remove(

51
compiler/rustc_infer/src/infer/outlives/obligations.rs
View file

@ -76,23 +76,56 @@ use crate::infer::outlives::env::RegionBoundPairs;
use crate::infer::outlives::verify::VerifyBoundCx;
use crate::infer::resolve::OpportunisticRegionResolver;
use crate::infer::snapshot::undo_log::UndoLog;
use crate::infer::{self, GenericKind, InferCtxt, RegionObligation, SubregionOrigin, VerifyBound};
use crate::infer::{
self, GenericKind, InferCtxt, SubregionOrigin, TypeOutlivesConstraint, VerifyBound,
};
use crate::traits::{ObligationCause, ObligationCauseCode};
impl<'tcx> InferCtxt<'tcx> {
pub fn register_outlives_constraint(
&self,
ty::OutlivesPredicate(arg, r2): ty::OutlivesPredicate<'tcx, ty::GenericArg<'tcx>>,
cause: &ObligationCause<'tcx>,
) {
match arg.unpack() {
ty::GenericArgKind::Lifetime(r1) => {
self.register_region_outlives_constraint(ty::OutlivesPredicate(r1, r2), cause);
}
ty::GenericArgKind::Type(ty1) => {
self.register_type_outlives_constraint(ty1, r2, cause);
}
ty::GenericArgKind::Const(_) => unreachable!(),
}
}
pub fn register_region_outlives_constraint(
&self,
ty::OutlivesPredicate(r_a, r_b): ty::RegionOutlivesPredicate<'tcx>,
cause: &ObligationCause<'tcx>,
) {
let origin = SubregionOrigin::from_obligation_cause(cause, || {
SubregionOrigin::RelateRegionParamBound(cause.span, None)
});
// `'a: 'b` ==> `'b <= 'a`
self.sub_regions(origin, r_b, r_a);
}
/// Registers that the given region obligation must be resolved
/// from within the scope of `body_id`. These regions are enqueued
/// and later processed by regionck, when full type information is
/// available (see `region_obligations` field for more
/// information).
#[instrument(level = "debug", skip(self))]
pub fn register_region_obligation(&self, obligation: RegionObligation<'tcx>) {
pub fn register_type_outlives_constraint_inner(
&self,
obligation: TypeOutlivesConstraint<'tcx>,
) {
let mut inner = self.inner.borrow_mut();
inner.undo_log.push(UndoLog::PushRegionObligation);
inner.undo_log.push(UndoLog::PushTypeOutlivesConstraint);
inner.region_obligations.push(obligation);
}
pub fn register_region_obligation_with_cause(
pub fn register_type_outlives_constraint(
&self,
sup_type: Ty<'tcx>,
sub_region: Region<'tcx>,
@ -124,11 +157,15 @@ impl<'tcx> InferCtxt<'tcx> {
)
});
self.register_region_obligation(RegionObligation { sup_type, sub_region, origin });
self.register_type_outlives_constraint_inner(TypeOutlivesConstraint {
sup_type,
sub_region,
origin,
});
}
/// Trait queries just want to pass back type obligations "as is"
pub fn take_registered_region_obligations(&self) -> Vec<RegionObligation<'tcx>> {
pub fn take_registered_region_obligations(&self) -> Vec<TypeOutlivesConstraint<'tcx>> {
std::mem::take(&mut self.inner.borrow_mut().region_obligations)
}
@ -166,7 +203,7 @@ impl<'tcx> InferCtxt<'tcx> {
);
}
for RegionObligation { sup_type, sub_region, origin } in my_region_obligations {
for TypeOutlivesConstraint { sup_type, sub_region, origin } in my_region_obligations {
let outlives = ty::Binder::dummy(ty::OutlivesPredicate(sup_type, sub_region));
let ty::OutlivesPredicate(sup_type, sub_region) =
deeply_normalize_ty(outlives, origin.clone())

4
compiler/rustc_infer/src/infer/snapshot/undo_log.rs
View file

@ -26,7 +26,7 @@ pub(crate) enum UndoLog<'tcx> {
RegionConstraintCollector(region_constraints::UndoLog<'tcx>),
RegionUnificationTable(sv::UndoLog<ut::Delegate<RegionVidKey<'tcx>>>),
ProjectionCache(traits::UndoLog<'tcx>),
PushRegionObligation,
PushTypeOutlivesConstraint,
}
macro_rules! impl_from {
@ -72,7 +72,7 @@ impl<'tcx> Rollback<UndoLog<'tcx>> for InferCtxtInner<'tcx> {
self.region_constraint_storage.as_mut().unwrap().unification_table.reverse(undo)
}
UndoLog::ProjectionCache(undo) => self.projection_cache.reverse(undo),
UndoLog::PushRegionObligation => {
UndoLog::PushTypeOutlivesConstraint => {
self.region_obligations.pop();
}
}

2
compiler/rustc_lint/src/early.rs
View file

@ -277,7 +277,7 @@ impl<'ast, 'ecx, 'tcx, T: EarlyLintPass> ast_visit::Visitor<'ast>
ast_visit::walk_attribute(self, attr);
}
fn visit_mac_def(&mut self, mac: &'ast ast::MacroDef, id: ast::NodeId) {
fn visit_macro_def(&mut self, mac: &'ast ast::MacroDef, id: ast::NodeId) {
lint_callback!(self, check_mac_def, mac);
self.check_id(id);
}

35
compiler/rustc_lint/src/unused.rs
View file

@ -1,8 +1,7 @@
use std::iter;
use rustc_ast as ast;
use rustc_ast::util::{classify, parser};
use rustc_ast::{ExprKind, StmtKind};
use rustc_ast::{self as ast, ExprKind, HasAttrs as _, StmtKind};
use rustc_errors::{MultiSpan, pluralize};
use rustc_hir::def::{DefKind, Res};
use rustc_hir::def_id::DefId;
@ -780,26 +779,30 @@ trait UnusedDelimLint {
right_pos: Option<BytePos>,
is_kw: bool,
) {
let spans = match value.kind {
ast::ExprKind::Block(ref block, None) if let [stmt] = block.stmts.as_slice() => stmt
.span
.find_ancestor_inside(value.span)
.map(|span| (value.span.with_hi(span.lo()), value.span.with_lo(span.hi()))),
let span_with_attrs = match value.kind {
ast::ExprKind::Block(ref block, None) if let [stmt] = block.stmts.as_slice() => {
// For the statements with attributes, like `{ #[allow()] println!("Hello!") }`,
// the span should contains the attributes, or the suggestion will remove them.
if let Some(attr_lo) = stmt.attrs().iter().map(|attr| attr.span.lo()).min() {
stmt.span.with_lo(attr_lo)
} else {
stmt.span
}
}
ast::ExprKind::Paren(ref expr) => {
// For the expr with attributes, like `let _ = (#[inline] || println!("Hello!"));`,
// the span should contains the attributes, or the suggestion will remove them.
let expr_span_with_attrs =
if let Some(attr_lo) = expr.attrs.iter().map(|attr| attr.span.lo()).min() {
expr.span.with_lo(attr_lo)
} else {
expr.span
};
expr_span_with_attrs.find_ancestor_inside(value.span).map(|expr_span| {
(value.span.with_hi(expr_span.lo()), value.span.with_lo(expr_span.hi()))
})
if let Some(attr_lo) = expr.attrs.iter().map(|attr| attr.span.lo()).min() {
expr.span.with_lo(attr_lo)
} else {
expr.span
}
}
_ => return,
};
let spans = span_with_attrs
.find_ancestor_inside(value.span)
.map(|span| (value.span.with_hi(span.lo()), value.span.with_lo(span.hi())));
let keep_space = (
left_pos.is_some_and(|s| s >= value.span.lo()),
right_pos.is_some_and(|s| s <= value.span.hi()),

3
compiler/rustc_middle/src/hir/place.rs
View file

@ -43,6 +43,9 @@ pub enum ProjectionKind {
///
/// This is unused if `-Znext-solver` is enabled.
OpaqueCast,
/// `unwrap_binder!(expr)`
UnwrapUnsafeBinder,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, TyEncodable, TyDecodable, HashStable)]

58
compiler/rustc_middle/src/mir/interpret/allocation.rs
View file

@ -27,12 +27,21 @@ use crate::ty;
/// Functionality required for the bytes of an `Allocation`.
pub trait AllocBytes: Clone + fmt::Debug + Deref<Target = [u8]> + DerefMut<Target = [u8]> {
/// The type of extra parameters passed in when creating an allocation.
/// Can be used by `interpret::Machine` instances to make runtime-configuration-dependent
/// decisions about the allocation strategy.
type AllocParams;
/// Create an `AllocBytes` from a slice of `u8`.
fn from_bytes<'a>(slice: impl Into<Cow<'a, [u8]>>, _align: Align) -> Self;
fn from_bytes<'a>(
slice: impl Into<Cow<'a, [u8]>>,
_align: Align,
_params: Self::AllocParams,
) -> Self;
/// Create a zeroed `AllocBytes` of the specified size and alignment.
/// Returns `None` if we ran out of memory on the host.
fn zeroed(size: Size, _align: Align) -> Option<Self>;
fn zeroed(size: Size, _align: Align, _params: Self::AllocParams) -> Option<Self>;
/// Gives direct access to the raw underlying storage.
///
@ -51,11 +60,13 @@ pub trait AllocBytes: Clone + fmt::Debug + Deref<Target = [u8]> + DerefMut<Targe
/// Default `bytes` for `Allocation` is a `Box<u8>`.
impl AllocBytes for Box<[u8]> {
fn from_bytes<'a>(slice: impl Into<Cow<'a, [u8]>>, _align: Align) -> Self {
type AllocParams = ();
fn from_bytes<'a>(slice: impl Into<Cow<'a, [u8]>>, _align: Align, _params: ()) -> Self {
Box::<[u8]>::from(slice.into())
}
fn zeroed(size: Size, _align: Align) -> Option<Self> {
fn zeroed(size: Size, _align: Align, _params: ()) -> Option<Self> {
let bytes = Box::<[u8]>::try_new_zeroed_slice(size.bytes().try_into().ok()?).ok()?;
// SAFETY: the box was zero-allocated, which is a valid initial value for Box<[u8]>
let bytes = unsafe { bytes.assume_init() };
@ -172,9 +183,8 @@ fn all_zero(buf: &[u8]) -> bool {
}
/// Custom encoder for [`Allocation`] to more efficiently represent the case where all bytes are 0.
impl<Prov: Provenance, Extra, Bytes, E: Encoder> Encodable<E> for Allocation<Prov, Extra, Bytes>
impl<Prov: Provenance, Extra, E: Encoder> Encodable<E> for Allocation<Prov, Extra, Box<[u8]>>
where
Bytes: AllocBytes,
ProvenanceMap<Prov>: Encodable<E>,
Extra: Encodable<E>,
{
@ -192,9 +202,8 @@ where
}
}
impl<Prov: Provenance, Extra, Bytes, D: Decoder> Decodable<D> for Allocation<Prov, Extra, Bytes>
impl<Prov: Provenance, Extra, D: Decoder> Decodable<D> for Allocation<Prov, Extra, Box<[u8]>>
where
Bytes: AllocBytes,
ProvenanceMap<Prov>: Decodable<D>,
Extra: Decodable<D>,
{
@ -203,7 +212,7 @@ where
let len = decoder.read_usize();
let bytes = if all_zero { vec![0u8; len] } else { decoder.read_raw_bytes(len).to_vec() };
let bytes = Bytes::from_bytes(bytes, align);
let bytes = <Box<[u8]> as AllocBytes>::from_bytes(bytes, align, ());
let provenance = Decodable::decode(decoder);
let init_mask = Decodable::decode(decoder);
@ -395,8 +404,9 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
slice: impl Into<Cow<'a, [u8]>>,
align: Align,
mutability: Mutability,
params: <Bytes as AllocBytes>::AllocParams,
) -> Self {
let bytes = Bytes::from_bytes(slice, align);
let bytes = Bytes::from_bytes(slice, align, params);
let size = Size::from_bytes(bytes.len());
Self {
bytes,
@ -408,14 +418,18 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
}
}
pub fn from_bytes_byte_aligned_immutable<'a>(slice: impl Into<Cow<'a, [u8]>>) -> Self {
Allocation::from_bytes(slice, Align::ONE, Mutability::Not)
pub fn from_bytes_byte_aligned_immutable<'a>(
slice: impl Into<Cow<'a, [u8]>>,
params: <Bytes as AllocBytes>::AllocParams,
) -> Self {
Allocation::from_bytes(slice, Align::ONE, Mutability::Not, params)
}
fn new_inner<R>(
size: Size,
align: Align,
init: AllocInit,
params: <Bytes as AllocBytes>::AllocParams,
fail: impl FnOnce() -> R,
) -> Result<Self, R> {
// We raise an error if we cannot create the allocation on the host.
@ -424,7 +438,7 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
// deterministic. However, we can be non-deterministic here because all uses of const
// evaluation (including ConstProp!) will make compilation fail (via hard error
// or ICE) upon encountering a `MemoryExhausted` error.
let bytes = Bytes::zeroed(size, align).ok_or_else(fail)?;
let bytes = Bytes::zeroed(size, align, params).ok_or_else(fail)?;
Ok(Allocation {
bytes,
@ -444,8 +458,13 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
/// Try to create an Allocation of `size` bytes, failing if there is not enough memory
/// available to the compiler to do so.
pub fn try_new<'tcx>(size: Size, align: Align, init: AllocInit) -> InterpResult<'tcx, Self> {
Self::new_inner(size, align, init, || {
pub fn try_new<'tcx>(
size: Size,
align: Align,
init: AllocInit,
params: <Bytes as AllocBytes>::AllocParams,
) -> InterpResult<'tcx, Self> {
Self::new_inner(size, align, init, params, || {
ty::tls::with(|tcx| tcx.dcx().delayed_bug("exhausted memory during interpretation"));
InterpErrorKind::ResourceExhaustion(ResourceExhaustionInfo::MemoryExhausted)
})
@ -457,8 +476,13 @@ impl<Prov: Provenance, Bytes: AllocBytes> Allocation<Prov, (), Bytes> {
///
/// Example use case: To obtain an Allocation filled with specific data,
/// first call this function and then call write_scalar to fill in the right data.
pub fn new(size: Size, align: Align, init: AllocInit) -> Self {
match Self::new_inner(size, align, init, || {
pub fn new(
size: Size,
align: Align,
init: AllocInit,
params: <Bytes as AllocBytes>::AllocParams,
) -> Self {
match Self::new_inner(size, align, init, params, || {
panic!(
"interpreter ran out of memory: cannot create allocation of {} bytes",
size.bytes()

2
compiler/rustc_middle/src/ty/context.rs
View file

@ -1582,7 +1582,7 @@ impl<'tcx> TyCtxt<'tcx> {
/// Returns the same `AllocId` if called again with the same bytes.
pub fn allocate_bytes_dedup(self, bytes: &[u8], salt: usize) -> interpret::AllocId {
// Create an allocation that just contains these bytes.
let alloc = interpret::Allocation::from_bytes_byte_aligned_immutable(bytes);
let alloc = interpret::Allocation::from_bytes_byte_aligned_immutable(bytes, ());
let alloc = self.mk_const_alloc(alloc);
self.reserve_and_set_memory_dedup(alloc, salt)
}

2
compiler/rustc_middle/src/ty/vtable.rs
View file

@ -110,7 +110,7 @@ pub(super) fn vtable_allocation_provider<'tcx>(
let ptr_align = tcx.data_layout.pointer_align.abi;
let vtable_size = ptr_size * u64::try_from(vtable_entries.len()).unwrap();
let mut vtable = Allocation::new(vtable_size, ptr_align, AllocInit::Uninit);
let mut vtable = Allocation::new(vtable_size, ptr_align, AllocInit::Uninit, ());
// No need to do any alignment checks on the memory accesses below, because we know the
// allocation is correctly aligned as we created it above. Also we're only offsetting by

6
compiler/rustc_mir_build/src/builder/expr/as_constant.rs
View file

@ -121,14 +121,14 @@ fn lit_to_mir_constant<'tcx>(tcx: TyCtxt<'tcx>, lit_input: LitToConstInput<'tcx>
let value = match (lit, lit_ty.kind()) {
(ast::LitKind::Str(s, _), ty::Ref(_, inner_ty, _)) if inner_ty.is_str() => {
let s = s.as_str();
let allocation = Allocation::from_bytes_byte_aligned_immutable(s.as_bytes());
let allocation = Allocation::from_bytes_byte_aligned_immutable(s.as_bytes(), ());
let allocation = tcx.mk_const_alloc(allocation);
ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
}
(ast::LitKind::ByteStr(data, _), ty::Ref(_, inner_ty, _))
if matches!(inner_ty.kind(), ty::Slice(_)) =>
{
let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]);
let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8], ());
let allocation = tcx.mk_const_alloc(allocation);
ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
}
@ -138,7 +138,7 @@ fn lit_to_mir_constant<'tcx>(tcx: TyCtxt<'tcx>, lit_input: LitToConstInput<'tcx>
}
(ast::LitKind::CStr(data, _), ty::Ref(_, inner_ty, _)) if matches!(inner_ty.kind(), ty::Adt(def, _) if tcx.is_lang_item(def.did(), LangItem::CStr)) =>
{
let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8]);
let allocation = Allocation::from_bytes_byte_aligned_immutable(data as &[u8], ());
let allocation = tcx.mk_const_alloc(allocation);
ConstValue::Slice { data: allocation, meta: allocation.inner().size().bytes() }
}

3
compiler/rustc_mir_build/src/builder/expr/as_place.rs
View file

@ -240,6 +240,9 @@ fn strip_prefix<'tcx>(
HirProjectionKind::OpaqueCast => {
assert_matches!(iter.next(), Some(ProjectionElem::OpaqueCast(..)));
}
HirProjectionKind::UnwrapUnsafeBinder => {
assert_matches!(iter.next(), Some(ProjectionElem::UnwrapUnsafeBinder(..)));
}
HirProjectionKind::Index | HirProjectionKind::Subslice => {
bug!("unexpected projection kind: {:?}", projection);
}

3
compiler/rustc_mir_build/src/thir/cx/expr.rs
View file

@ -1220,6 +1220,9 @@ impl<'tcx> ThirBuildCx<'tcx> {
HirProjectionKind::OpaqueCast => {
ExprKind::Use { source: self.thir.exprs.push(captured_place_expr) }
}
HirProjectionKind::UnwrapUnsafeBinder => ExprKind::PlaceUnwrapUnsafeBinder {
source: self.thir.exprs.push(captured_place_expr),
},
HirProjectionKind::Index | HirProjectionKind::Subslice => {
// We don't capture these projections, so we can ignore them here
continue;

17
compiler/rustc_mir_transform/src/elaborate_drop.rs
View file

@ -1278,6 +1278,23 @@ where
}
ty::Slice(ety) => self.drop_loop_trio_for_slice(*ety),
ty::UnsafeBinder(_) => {
// Unsafe binders may elaborate drops if their inner type isn't copy.
// This is enforced in typeck, so this should never happen.
self.tcx().dcx().span_delayed_bug(
self.source_info.span,
"open drop for unsafe binder shouldn't be encountered",
);
self.elaborator.patch().new_block(BasicBlockData {
statements: vec![],
terminator: Some(Terminator {
source_info: self.source_info,
kind: TerminatorKind::Unreachable,
}),
is_cleanup: self.unwind.is_cleanup(),
})
}
_ => span_bug!(self.source_info.span, "open drop from non-ADT `{:?}`", ty),
}
}

1
compiler/rustc_mir_transform/src/large_enums.rs
View file

@ -241,6 +241,7 @@ impl EnumSizeOpt {
data,
tcx.data_layout.ptr_sized_integer().align(&tcx.data_layout).abi,
Mutability::Not,
(),
);
let alloc = tcx.reserve_and_set_memory_alloc(tcx.mk_const_alloc(alloc));
Some((*adt_def, num_discrs, *alloc_cache.entry(ty).or_insert(alloc)))

59
compiler/rustc_next_trait_solver/src/canonicalizer.rs
View file

@ -4,12 +4,22 @@ use rustc_type_ir::data_structures::{HashMap, ensure_sufficient_stack};
use rustc_type_ir::inherent::*;
use rustc_type_ir::solve::{Goal, QueryInput};
use rustc_type_ir::{
self as ty, Canonical, CanonicalTyVarKind, CanonicalVarKind, InferCtxtLike, Interner,
TypeFoldable, TypeFolder, TypeSuperFoldable, TypeVisitableExt,
self as ty, Canonical, CanonicalTyVarKind, CanonicalVarKind, Flags, InferCtxtLike, Interner,
TypeFlags, TypeFoldable, TypeFolder, TypeSuperFoldable, TypeVisitableExt,
};
use crate::delegate::SolverDelegate;
/// Does this have infer/placeholder/param, free regions or ReErased?
const NEEDS_CANONICAL: TypeFlags = TypeFlags::from_bits(
TypeFlags::HAS_INFER.bits()
| TypeFlags::HAS_PLACEHOLDER.bits()
| TypeFlags::HAS_PARAM.bits()
| TypeFlags::HAS_FREE_REGIONS.bits()
| TypeFlags::HAS_RE_ERASED.bits(),
)
.unwrap();
/// Whether we're canonicalizing a query input or the query response.
///
/// When canonicalizing an input we're in the context of the caller
@ -79,7 +89,11 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
cache: Default::default(),
};
let value = value.fold_with(&mut canonicalizer);
let value = if value.has_type_flags(NEEDS_CANONICAL) {
value.fold_with(&mut canonicalizer)
} else {
value
};
assert!(!value.has_infer(), "unexpected infer in {value:?}");
assert!(!value.has_placeholders(), "unexpected placeholders in {value:?}");
let (max_universe, variables) = canonicalizer.finalize();
@ -111,7 +125,14 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
cache: Default::default(),
};
let param_env = input.goal.param_env.fold_with(&mut env_canonicalizer);
let param_env = input.goal.param_env;
let param_env = if param_env.has_type_flags(NEEDS_CANONICAL) {
param_env.fold_with(&mut env_canonicalizer)
} else {
param_env
};
debug_assert_eq!(env_canonicalizer.binder_index, ty::INNERMOST);
// Then canonicalize the rest of the input without keeping `'static`
// while *mostly* reusing the canonicalizer from above.
@ -134,10 +155,22 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
cache: Default::default(),
};
let predicate = input.goal.predicate.fold_with(&mut rest_canonicalizer);
let predicate = input.goal.predicate;
let predicate = if predicate.has_type_flags(NEEDS_CANONICAL) {
predicate.fold_with(&mut rest_canonicalizer)
} else {
predicate
};
let goal = Goal { param_env, predicate };
let predefined_opaques_in_body = input.predefined_opaques_in_body;
let predefined_opaques_in_body =
input.predefined_opaques_in_body.fold_with(&mut rest_canonicalizer);
if input.predefined_opaques_in_body.has_type_flags(NEEDS_CANONICAL) {
predefined_opaques_in_body.fold_with(&mut rest_canonicalizer)
} else {
predefined_opaques_in_body
};
let value = QueryInput { goal, predefined_opaques_in_body };
assert!(!value.has_infer(), "unexpected infer in {value:?}");
@ -387,7 +420,11 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
| ty::Alias(_, _)
| ty::Bound(_, _)
| ty::Error(_) => {
return ensure_sufficient_stack(|| t.super_fold_with(self));
return if t.has_type_flags(NEEDS_CANONICAL) {
ensure_sufficient_stack(|| t.super_fold_with(self))
} else {
t
};
}
};
@ -522,11 +559,17 @@ impl<D: SolverDelegate<Interner = I>, I: Interner> TypeFolder<I> for Canonicaliz
| ty::ConstKind::Unevaluated(_)
| ty::ConstKind::Value(_)
| ty::ConstKind::Error(_)
| ty::ConstKind::Expr(_) => return c.super_fold_with(self),
| ty::ConstKind::Expr(_) => {
return if c.has_type_flags(NEEDS_CANONICAL) { c.super_fold_with(self) } else { c };
}
};
let var = self.get_or_insert_bound_var(c, kind);
Const::new_anon_bound(self.cx(), self.binder_index, var)
}
fn fold_predicate(&mut self, p: I::Predicate) -> I::Predicate {
if p.flags().intersects(NEEDS_CANONICAL) { p.super_fold_with(self) } else { p }
}
}

4
compiler/rustc_next_trait_solver/src/resolve.rs
View file

@ -86,4 +86,8 @@ impl<D: SolverDelegate<Interner = I>, I: Interner> TypeFolder<I> for EagerResolv
}
}
}
fn fold_predicate(&mut self, p: I::Predicate) -> I::Predicate {
if p.has_infer() { p.super_fold_with(self) } else { p }
}
}

15
compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs
View file

@ -53,10 +53,10 @@ where
{
/// Canonicalizes the goal remembering the original values
/// for each bound variable.
pub(super) fn canonicalize_goal<T: TypeFoldable<I>>(
pub(super) fn canonicalize_goal(
&self,
goal: Goal<I, T>,
) -> (Vec<I::GenericArg>, CanonicalInput<I, T>) {
goal: Goal<I, I::Predicate>,
) -> (Vec<I::GenericArg>, CanonicalInput<I, I::Predicate>) {
// We only care about one entry per `OpaqueTypeKey` here,
// so we only canonicalize the lookup table and ignore
// duplicate entries.
@ -130,7 +130,12 @@ where
if goals.is_empty() {
assert!(matches!(goals_certainty, Certainty::Yes));
}
(Certainty::Yes, NestedNormalizationGoals(goals))
(
Certainty::Yes,
NestedNormalizationGoals(
goals.into_iter().map(|(s, g, _)| (s, g)).collect(),
),
)
}
_ => {
let certainty = shallow_certainty.and(goals_certainty);
@ -272,7 +277,7 @@ where
pub(super) fn instantiate_and_apply_query_response(
&mut self,
param_env: I::ParamEnv,
original_values: Vec<I::GenericArg>,
original_values: &[I::GenericArg],
response: CanonicalResponse<I>,
) -> (NestedNormalizationGoals<I>, Certainty) {
let instantiation = Self::compute_query_response_instantiation_values(

131
compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs
View file

@ -22,8 +22,9 @@ use crate::delegate::SolverDelegate;
use crate::solve::inspect::{self, ProofTreeBuilder};
use crate::solve::search_graph::SearchGraph;
use crate::solve::{
CanonicalInput, Certainty, FIXPOINT_STEP_LIMIT, Goal, GoalEvaluationKind, GoalSource,
HasChanged, NestedNormalizationGoals, NoSolution, QueryInput, QueryResult,
CanonicalInput, Certainty, FIXPOINT_STEP_LIMIT, Goal, GoalEvaluation, GoalEvaluationKind,
GoalSource, GoalStalledOn, HasChanged, NestedNormalizationGoals, NoSolution, QueryInput,
QueryResult,
};
pub(super) mod canonical;
@ -115,7 +116,7 @@ where
pub(super) search_graph: &'a mut SearchGraph<D>,
nested_goals: Vec<(GoalSource, Goal<I, I::Predicate>)>,
nested_goals: Vec<(GoalSource, Goal<I, I::Predicate>, Option<GoalStalledOn<I>>)>,
pub(super) origin_span: I::Span,
@ -147,8 +148,9 @@ pub trait SolverDelegateEvalExt: SolverDelegate {
goal: Goal<Self::Interner, <Self::Interner as Interner>::Predicate>,
generate_proof_tree: GenerateProofTree,
span: <Self::Interner as Interner>::Span,
stalled_on: Option<GoalStalledOn<Self::Interner>>,
) -> (
Result<(HasChanged, Certainty), NoSolution>,
Result<GoalEvaluation<Self::Interner>, NoSolution>,
Option<inspect::GoalEvaluation<Self::Interner>>,
);
@ -171,8 +173,12 @@ pub trait SolverDelegateEvalExt: SolverDelegate {
&self,
goal: Goal<Self::Interner, <Self::Interner as Interner>::Predicate>,
generate_proof_tree: GenerateProofTree,
stalled_on: Option<GoalStalledOn<Self::Interner>>,
) -> (
Result<(NestedNormalizationGoals<Self::Interner>, HasChanged, Certainty), NoSolution>,
Result<
(NestedNormalizationGoals<Self::Interner>, GoalEvaluation<Self::Interner>),
NoSolution,
>,
Option<inspect::GoalEvaluation<Self::Interner>>,
);
}
@ -188,9 +194,10 @@ where
goal: Goal<I, I::Predicate>,
generate_proof_tree: GenerateProofTree,
span: I::Span,
) -> (Result<(HasChanged, Certainty), NoSolution>, Option<inspect::GoalEvaluation<I>>) {
stalled_on: Option<GoalStalledOn<I>>,
) -> (Result<GoalEvaluation<I>, NoSolution>, Option<inspect::GoalEvaluation<I>>) {
EvalCtxt::enter_root(self, self.cx().recursion_limit(), generate_proof_tree, span, |ecx| {
ecx.evaluate_goal(GoalEvaluationKind::Root, GoalSource::Misc, goal)
ecx.evaluate_goal(GoalEvaluationKind::Root, GoalSource::Misc, goal, stalled_on)
})
}
@ -201,7 +208,7 @@ where
) -> bool {
self.probe(|| {
EvalCtxt::enter_root(self, root_depth, GenerateProofTree::No, I::Span::dummy(), |ecx| {
ecx.evaluate_goal(GoalEvaluationKind::Root, GoalSource::Misc, goal)
ecx.evaluate_goal(GoalEvaluationKind::Root, GoalSource::Misc, goal, None)
})
.0
})
@ -213,8 +220,9 @@ where
&self,
goal: Goal<I, I::Predicate>,
generate_proof_tree: GenerateProofTree,
stalled_on: Option<GoalStalledOn<I>>,
) -> (
Result<(NestedNormalizationGoals<I>, HasChanged, Certainty), NoSolution>,
Result<(NestedNormalizationGoals<I>, GoalEvaluation<I>), NoSolution>,
Option<inspect::GoalEvaluation<I>>,
) {
EvalCtxt::enter_root(
@ -222,7 +230,9 @@ where
self.cx().recursion_limit(),
generate_proof_tree,
I::Span::dummy(),
|ecx| ecx.evaluate_goal_raw(GoalEvaluationKind::Root, GoalSource::Misc, goal),
|ecx| {
ecx.evaluate_goal_raw(GoalEvaluationKind::Root, GoalSource::Misc, goal, stalled_on)
},
)
}
}
@ -447,11 +457,12 @@ where
goal_evaluation_kind: GoalEvaluationKind,
source: GoalSource,
goal: Goal<I, I::Predicate>,
) -> Result<(HasChanged, Certainty), NoSolution> {
let (normalization_nested_goals, has_changed, certainty) =
self.evaluate_goal_raw(goal_evaluation_kind, source, goal)?;
stalled_on: Option<GoalStalledOn<I>>,
) -> Result<GoalEvaluation<I>, NoSolution> {
let (normalization_nested_goals, goal_evaluation) =
self.evaluate_goal_raw(goal_evaluation_kind, source, goal, stalled_on)?;
assert!(normalization_nested_goals.is_empty());
Ok((has_changed, certainty))
Ok(goal_evaluation)
}
/// Recursively evaluates `goal`, returning the nested goals in case
@ -466,7 +477,29 @@ where
goal_evaluation_kind: GoalEvaluationKind,
source: GoalSource,
goal: Goal<I, I::Predicate>,
) -> Result<(NestedNormalizationGoals<I>, HasChanged, Certainty), NoSolution> {
stalled_on: Option<GoalStalledOn<I>>,
) -> Result<(NestedNormalizationGoals<I>, GoalEvaluation<I>), NoSolution> {
// If we have run this goal before, and it was stalled, check that any of the goal's
// args have changed. Otherwise, we don't need to re-run the goal because it'll remain
// stalled, since it'll canonicalize the same way and evaluation is pure.
if let Some(stalled_on) = stalled_on {
if !stalled_on.stalled_vars.iter().any(|value| self.delegate.is_changed_arg(*value))
&& !self
.delegate
.opaque_types_storage_num_entries()
.needs_reevaluation(stalled_on.num_opaques)
{
return Ok((
NestedNormalizationGoals::empty(),
GoalEvaluation {
certainty: Certainty::Maybe(stalled_on.stalled_cause),
has_changed: HasChanged::No,
stalled_on: Some(stalled_on),
},
));
}
}
let (orig_values, canonical_goal) = self.canonicalize_goal(goal);
let mut goal_evaluation =
self.inspect.new_goal_evaluation(goal, &orig_values, goal_evaluation_kind);
@ -489,7 +522,7 @@ where
if !has_only_region_constraints(response) { HasChanged::Yes } else { HasChanged::No };
let (normalization_nested_goals, certainty) =
self.instantiate_and_apply_query_response(goal.param_env, orig_values, response);
self.instantiate_and_apply_query_response(goal.param_env, &orig_values, response);
self.inspect.goal_evaluation(goal_evaluation);
// FIXME: We previously had an assert here that checked that recomputing
@ -502,7 +535,42 @@ where
// Once we have decided on how to handle trait-system-refactor-initiative#75,
// we should re-add an assert here.
Ok((normalization_nested_goals, has_changed, certainty))
let stalled_on = match certainty {
Certainty::Yes => None,
Certainty::Maybe(stalled_cause) => match has_changed {
// FIXME: We could recompute a *new* set of stalled variables by walking
// through the orig values, resolving, and computing the root vars of anything
// that is not resolved. Only when *these* have changed is it meaningful
// to recompute this goal.
HasChanged::Yes => None,
HasChanged::No => {
// Remove the unconstrained RHS arg, which is expected to have changed.
let mut stalled_vars = orig_values;
if let Some(normalizes_to) = goal.predicate.as_normalizes_to() {
let normalizes_to = normalizes_to.skip_binder();
let rhs_arg: I::GenericArg = normalizes_to.term.into();
let idx = stalled_vars
.iter()
.rposition(|arg| *arg == rhs_arg)
.expect("expected unconstrained arg");
stalled_vars.swap_remove(idx);
}
Some(GoalStalledOn {
num_opaques: canonical_goal
.canonical
.value
.predefined_opaques_in_body
.opaque_types
.len(),
stalled_vars,
stalled_cause,
})
}
},
};
Ok((normalization_nested_goals, GoalEvaluation { certainty, has_changed, stalled_on }))
}
fn compute_goal(&mut self, goal: Goal<I, I::Predicate>) -> QueryResult<I> {
@ -602,7 +670,7 @@ where
let cx = self.cx();
// If this loop did not result in any progress, what's our final certainty.
let mut unchanged_certainty = Some(Certainty::Yes);
for (source, goal) in mem::take(&mut self.nested_goals) {
for (source, goal, stalled_on) in mem::take(&mut self.nested_goals) {
if let Some(has_changed) = self.delegate.compute_goal_fast_path(goal, self.origin_span)
{
if matches!(has_changed, HasChanged::Yes) {
@ -630,11 +698,18 @@ where
let unconstrained_goal =
goal.with(cx, ty::NormalizesTo { alias: pred.alias, term: unconstrained_rhs });
let (NestedNormalizationGoals(nested_goals), _, certainty) =
self.evaluate_goal_raw(GoalEvaluationKind::Nested, source, unconstrained_goal)?;
let (
NestedNormalizationGoals(nested_goals),
GoalEvaluation { certainty, stalled_on, has_changed: _ },
) = self.evaluate_goal_raw(
GoalEvaluationKind::Nested,
source,
unconstrained_goal,
stalled_on,
)?;
// Add the nested goals from normalization to our own nested goals.
trace!(?nested_goals);
self.nested_goals.extend(nested_goals);
self.nested_goals.extend(nested_goals.into_iter().map(|(s, g)| (s, g, None)));
// Finally, equate the goal's RHS with the unconstrained var.
//
@ -660,6 +735,8 @@ where
// looking at the "has changed" return from evaluate_goal,
// because we expect the `unconstrained_rhs` part of the predicate
// to have changed -- that means we actually normalized successfully!
// FIXME: Do we need to eagerly resolve here? Or should we check
// if the cache key has any changed vars?
let with_resolved_vars = self.resolve_vars_if_possible(goal);
if pred.alias != goal.predicate.as_normalizes_to().unwrap().skip_binder().alias {
unchanged_certainty = None;
@ -668,13 +745,13 @@ where
match certainty {
Certainty::Yes => {}
Certainty::Maybe(_) => {
self.nested_goals.push((source, with_resolved_vars));
self.nested_goals.push((source, with_resolved_vars, stalled_on));
unchanged_certainty = unchanged_certainty.map(|c| c.and(certainty));
}
}
} else {
let (has_changed, certainty) =
self.evaluate_goal(GoalEvaluationKind::Nested, source, goal)?;
let GoalEvaluation { certainty, has_changed, stalled_on } =
self.evaluate_goal(GoalEvaluationKind::Nested, source, goal, stalled_on)?;
if has_changed == HasChanged::Yes {
unchanged_certainty = None;
}
@ -682,7 +759,7 @@ where
match certainty {
Certainty::Yes => {}
Certainty::Maybe(_) => {
self.nested_goals.push((source, goal));
self.nested_goals.push((source, goal, stalled_on));
unchanged_certainty = unchanged_certainty.map(|c| c.and(certainty));
}
}
@ -706,7 +783,7 @@ where
goal.predicate =
goal.predicate.fold_with(&mut ReplaceAliasWithInfer::new(self, source, goal.param_env));
self.inspect.add_goal(self.delegate, self.max_input_universe, source, goal);
self.nested_goals.push((source, goal));
self.nested_goals.push((source, goal, None));
}
#[instrument(level = "trace", skip(self, goals))]
@ -1024,7 +1101,7 @@ where
}
pub(super) fn register_region_outlives(&self, a: I::Region, b: I::Region) {
// `b : a` ==> `a <= b`
// `'a: 'b` ==> `'b <= 'a`
self.delegate.sub_regions(b, a, self.origin_span);
}

19
compiler/rustc_next_trait_solver/src/solve/mod.rs
View file

@ -21,6 +21,7 @@ mod project_goals;
mod search_graph;
mod trait_goals;
use derive_where::derive_where;
use rustc_type_ir::inherent::*;
pub use rustc_type_ir::solve::*;
use rustc_type_ir::{self as ty, Interner, TypingMode};
@ -369,3 +370,21 @@ fn response_no_constraints_raw<I: Interner>(
},
}
}
/// The result of evaluating a goal.
pub struct GoalEvaluation<I: Interner> {
pub certainty: Certainty,
pub has_changed: HasChanged,
/// If the [`Certainty`] was `Maybe`, then keep track of whether the goal has changed
/// before rerunning it.
pub stalled_on: Option<GoalStalledOn<I>>,
}
/// The conditions that must change for a goal to warrant
#[derive_where(Clone, Debug; I: Interner)]
pub struct GoalStalledOn<I: Interner> {
pub num_opaques: usize,
pub stalled_vars: Vec<I::GenericArg>,
/// The cause that will be returned on subsequent evaluations if this goal remains stalled.
pub stalled_cause: MaybeCause,
}

2
compiler/rustc_smir/src/rustc_smir/alloc.rs
View file

@ -48,6 +48,7 @@ pub(crate) fn try_new_allocation<'tcx>(
size,
layout.align.abi,
AllocInit::Uninit,
(),
);
allocation
.write_scalar(&tables.tcx, alloc_range(Size::ZERO, size), scalar)
@ -65,6 +66,7 @@ pub(crate) fn try_new_allocation<'tcx>(
layout.size,
layout.align.abi,
AllocInit::Uninit,
(),
);
allocation
.write_scalar(

1
compiler/rustc_target/src/spec/targets/aarch64_apple_darwin.rs
View file

@ -22,6 +22,7 @@ pub(crate) fn target() -> Target {
max_atomic_width: Some(128),
// FIXME: The leak sanitizer currently fails the tests, see #88132.
supported_sanitizers: SanitizerSet::ADDRESS | SanitizerSet::CFI | SanitizerSet::THREAD,
supports_xray: true,
..opts
},
}

1
compiler/rustc_target/src/spec/targets/x86_64_apple_darwin.rs
View file

@ -23,6 +23,7 @@ pub(crate) fn target() -> Target {
| SanitizerSet::CFI
| SanitizerSet::LEAK
| SanitizerSet::THREAD,
supports_xray: true,
..opts
},
}

2
compiler/rustc_trait_selection/src/solve/delegate.rs
View file

@ -76,7 +76,7 @@ impl<'tcx> rustc_next_trait_solver::delegate::SolverDelegate for SolverDelegate<
Some(HasChanged::No)
}
ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(outlives)) => {
self.0.register_region_obligation_with_cause(
self.0.register_type_outlives_constraint(
outlives.0,
outlives.1,
&ObligationCause::dummy_with_span(span),

112
compiler/rustc_trait_selection/src/solve/fulfill.rs
View file

@ -13,8 +13,11 @@ use rustc_middle::ty::{
self, DelayedSet, Ty, TyCtxt, TypeSuperVisitable, TypeVisitable, TypeVisitor, TypingMode,
};
use rustc_next_trait_solver::delegate::SolverDelegate as _;
use rustc_next_trait_solver::solve::{GenerateProofTree, HasChanged, SolverDelegateEvalExt as _};
use rustc_next_trait_solver::solve::{
GenerateProofTree, GoalEvaluation, GoalStalledOn, HasChanged, SolverDelegateEvalExt as _,
};
use rustc_span::Span;
use thin_vec::ThinVec;
use tracing::instrument;
use self::derive_errors::*;
@ -25,6 +28,10 @@ use crate::traits::{FulfillmentError, ScrubbedTraitError};
mod derive_errors;
// FIXME: Do we need to use a `ThinVec` here?
type PendingObligations<'tcx> =
ThinVec<(PredicateObligation<'tcx>, Option<GoalStalledOn<TyCtxt<'tcx>>>)>;
/// A trait engine using the new trait solver.
///
/// This is mostly identical to how `evaluate_all` works inside of the
@ -54,13 +61,17 @@ struct ObligationStorage<'tcx> {
/// We cannot eagerly return these as error so we instead store them here
/// to avoid recomputing them each time `select_where_possible` is called.
/// This also allows us to return the correct `FulfillmentError` for them.
overflowed: PredicateObligations<'tcx>,
pending: PredicateObligations<'tcx>,
overflowed: Vec<PredicateObligation<'tcx>>,
pending: PendingObligations<'tcx>,
}
impl<'tcx> ObligationStorage<'tcx> {
fn register(&mut self, obligation: PredicateObligation<'tcx>) {
self.pending.push(obligation);
fn register(
&mut self,
obligation: PredicateObligation<'tcx>,
stalled_on: Option<GoalStalledOn<TyCtxt<'tcx>>>,
) {
self.pending.push((obligation, stalled_on));
}
fn has_pending_obligations(&self) -> bool {
@ -68,7 +79,8 @@ impl<'tcx> ObligationStorage<'tcx> {
}
fn clone_pending(&self) -> PredicateObligations<'tcx> {
let mut obligations = self.pending.clone();
let mut obligations: PredicateObligations<'tcx> =
self.pending.iter().map(|(o, _)| o.clone()).collect();
obligations.extend(self.overflowed.iter().cloned());
obligations
}
@ -76,8 +88,9 @@ impl<'tcx> ObligationStorage<'tcx> {
fn drain_pending(
&mut self,
cond: impl Fn(&PredicateObligation<'tcx>) -> bool,
) -> PredicateObligations<'tcx> {
let (unstalled, pending) = mem::take(&mut self.pending).into_iter().partition(cond);
) -> PendingObligations<'tcx> {
let (unstalled, pending) =
mem::take(&mut self.pending).into_iter().partition(|(o, _)| cond(o));
self.pending = pending;
unstalled
}
@ -90,13 +103,21 @@ impl<'tcx> ObligationStorage<'tcx> {
// we were to do another step of `select_where_possible`, which goals would
// change.
// FIXME: <https://github.com/Gankra/thin-vec/pull/66> is merged, this can be removed.
self.overflowed.extend(ExtractIf::new(&mut self.pending, |o| {
let goal = o.as_goal();
let result = <&SolverDelegate<'tcx>>::from(infcx)
.evaluate_root_goal(goal, GenerateProofTree::No, o.cause.span)
.0;
matches!(result, Ok((HasChanged::Yes, _)))
}));
self.overflowed.extend(
ExtractIf::new(&mut self.pending, |(o, stalled_on)| {
let goal = o.as_goal();
let result = <&SolverDelegate<'tcx>>::from(infcx)
.evaluate_root_goal(
goal,
GenerateProofTree::No,
o.cause.span,
stalled_on.take(),
)
.0;
matches!(result, Ok(GoalEvaluation { has_changed: HasChanged::Yes, .. }))
})
.map(|(o, _)| o),
);
})
}
}
@ -119,11 +140,11 @@ impl<'tcx, E: 'tcx> FulfillmentCtxt<'tcx, E> {
&self,
infcx: &InferCtxt<'tcx>,
obligation: &PredicateObligation<'tcx>,
result: &Result<(HasChanged, Certainty), NoSolution>,
result: &Result<GoalEvaluation<TyCtxt<'tcx>>, NoSolution>,
) {
if let Some(inspector) = infcx.obligation_inspector.get() {
let result = match result {
Ok((_, c)) => Ok(*c),
Ok(GoalEvaluation { certainty, .. }) => Ok(*certainty),
Err(NoSolution) => Err(NoSolution),
};
(inspector)(infcx, &obligation, result);
@ -142,14 +163,14 @@ where
obligation: PredicateObligation<'tcx>,
) {
assert_eq!(self.usable_in_snapshot, infcx.num_open_snapshots());
self.obligations.register(obligation);
self.obligations.register(obligation, None);
}
fn collect_remaining_errors(&mut self, infcx: &InferCtxt<'tcx>) -> Vec<E> {
self.obligations
.pending
.drain(..)
.map(|obligation| NextSolverError::Ambiguity(obligation))
.map(|(obligation, _)| NextSolverError::Ambiguity(obligation))
.chain(
self.obligations
.overflowed
@ -164,8 +185,8 @@ where
assert_eq!(self.usable_in_snapshot, infcx.num_open_snapshots());
let mut errors = Vec::new();
loop {
let mut has_changed = false;
for mut obligation in self.obligations.drain_pending(|_| true) {
let mut any_changed = false;
for (mut obligation, stalled_on) in self.obligations.drain_pending(|_| true) {
if !infcx.tcx.recursion_limit().value_within_limit(obligation.recursion_depth) {
self.obligations.on_fulfillment_overflow(infcx);
// Only return true errors that we have accumulated while processing.
@ -177,15 +198,20 @@ where
if let Some(fast_path_has_changed) =
delegate.compute_goal_fast_path(goal, obligation.cause.span)
{
has_changed |= matches!(fast_path_has_changed, HasChanged::Yes);
any_changed |= matches!(fast_path_has_changed, HasChanged::Yes);
continue;
}
let result = delegate
.evaluate_root_goal(goal, GenerateProofTree::No, obligation.cause.span)
.evaluate_root_goal(
goal,
GenerateProofTree::No,
obligation.cause.span,
stalled_on,
)
.0;
self.inspect_evaluated_obligation(infcx, &obligation, &result);
let (changed, certainty) = match result {
let GoalEvaluation { certainty, has_changed, stalled_on } = match result {
Ok(result) => result,
Err(NoSolution) => {
errors.push(E::from_solver_error(
@ -196,7 +222,7 @@ where
}
};
if changed == HasChanged::Yes {
if has_changed == HasChanged::Yes {
// We increment the recursion depth here to track the number of times
// this goal has resulted in inference progress. This doesn't precisely
// model the way that we track recursion depth in the old solver due
@ -204,16 +230,16 @@ where
// approximation and should only result in fulfillment overflow in
// pathological cases.
obligation.recursion_depth += 1;
has_changed = true;
any_changed = true;
}
match certainty {
Certainty::Yes => {}
Certainty::Maybe(_) => self.obligations.register(obligation),
Certainty::Maybe(_) => self.obligations.register(obligation, stalled_on),
}
}
if !has_changed {
if !any_changed {
break;
}
}
@ -247,20 +273,24 @@ where
return Default::default();
}
self.obligations.drain_pending(|obl| {
infcx.probe(|_| {
infcx
.visit_proof_tree(
obl.as_goal(),
&mut StalledOnCoroutines {
stalled_generators,
span: obl.cause.span,
cache: Default::default(),
},
)
.is_break()
self.obligations
.drain_pending(|obl| {
infcx.probe(|_| {
infcx
.visit_proof_tree(
obl.as_goal(),
&mut StalledOnCoroutines {
stalled_generators,
span: obl.cause.span,
cache: Default::default(),
},
)
.is_break()
})
})
})
.into_iter()
.map(|(o, _)| o)
.collect()
}
}

24
compiler/rustc_trait_selection/src/solve/fulfill/derive_errors.rs
View file

@ -11,7 +11,9 @@ use rustc_middle::traits::query::NoSolution;
use rustc_middle::ty::error::{ExpectedFound, TypeError};
use rustc_middle::ty::{self, Ty, TyCtxt};
use rustc_middle::{bug, span_bug};
use rustc_next_trait_solver::solve::{GenerateProofTree, SolverDelegateEvalExt as _};
use rustc_next_trait_solver::solve::{
GenerateProofTree, GoalEvaluation, SolverDelegateEvalExt as _,
};
use tracing::{instrument, trace};
use crate::solve::delegate::SolverDelegate;
@ -93,19 +95,21 @@ pub(super) fn fulfillment_error_for_stalled<'tcx>(
root_obligation.as_goal(),
GenerateProofTree::No,
root_obligation.cause.span,
None,
)
.0
{
Ok((_, Certainty::Maybe(MaybeCause::Ambiguity))) => {
Ok(GoalEvaluation { certainty: Certainty::Maybe(MaybeCause::Ambiguity), .. }) => {
(FulfillmentErrorCode::Ambiguity { overflow: None }, true)
}
Ok((
_,
Certainty::Maybe(MaybeCause::Overflow {
suggest_increasing_limit,
keep_constraints: _,
}),
)) => (
Ok(GoalEvaluation {
certainty:
Certainty::Maybe(MaybeCause::Overflow {
suggest_increasing_limit,
keep_constraints: _,
}),
..
}) => (
FulfillmentErrorCode::Ambiguity { overflow: Some(suggest_increasing_limit) },
// Don't look into overflows because we treat overflows weirdly anyways.
// We discard the inference constraints from overflowing goals, so
@ -115,7 +119,7 @@ pub(super) fn fulfillment_error_for_stalled<'tcx>(
// FIXME: We should probably just look into overflows here.
false,
),
Ok((_, Certainty::Yes)) => {
Ok(GoalEvaluation { certainty: Certainty::Yes, .. }) => {
bug!(
"did not expect successful goal when collecting ambiguity errors for `{:?}`",
infcx.resolve_vars_if_possible(root_obligation.predicate),

7
compiler/rustc_trait_selection/src/solve/inspect/analyse.rs
View file

@ -219,8 +219,8 @@ impl<'a, 'tcx> InspectCandidate<'a, 'tcx> {
// building their proof tree, the expected term was unconstrained, but when
// instantiating the candidate it is already constrained to the result of another
// candidate.
let proof_tree =
infcx.probe(|_| infcx.evaluate_root_goal_raw(goal, GenerateProofTree::Yes).1);
let proof_tree = infcx
.probe(|_| infcx.evaluate_root_goal_raw(goal, GenerateProofTree::Yes, None).1);
InspectGoal::new(
infcx,
self.goal.depth + 1,
@ -236,7 +236,7 @@ impl<'a, 'tcx> InspectCandidate<'a, 'tcx> {
// constraints, we get an ICE if we already applied the constraints
// from the chosen candidate.
let proof_tree = infcx
.probe(|_| infcx.evaluate_root_goal(goal, GenerateProofTree::Yes, span).1)
.probe(|_| infcx.evaluate_root_goal(goal, GenerateProofTree::Yes, span, None).1)
.unwrap();
InspectGoal::new(infcx, self.goal.depth + 1, proof_tree, None, source)
}
@ -442,6 +442,7 @@ impl<'tcx> InferCtxt<'tcx> {
goal,
GenerateProofTree::Yes,
visitor.span(),
None,
);
let proof_tree = proof_tree.unwrap();
visitor.visit_goal(&InspectGoal::new(self, depth, proof_tree, None, GoalSource::Misc))

1
compiler/rustc_trait_selection/src/solve/select.rs
View file

@ -15,6 +15,7 @@ use crate::solve::inspect::{self, ProofTreeInferCtxtExt};
#[extension(pub trait InferCtxtSelectExt<'tcx>)]
impl<'tcx> InferCtxt<'tcx> {
/// Do not use this directly. This is called from [`crate::traits::SelectionContext::select`].
fn select_in_new_trait_solver(
&self,
obligation: &TraitObligation<'tcx>,

8
compiler/rustc_trait_selection/src/traits/auto_trait.rs
View file

@ -726,7 +726,9 @@ impl<'tcx> AutoTraitFinder<'tcx> {
}
ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(binder)) => {
let binder = bound_predicate.rebind(binder);
selcx.infcx.region_outlives_predicate(&dummy_cause, binder)
selcx.infcx.enter_forall(binder, |pred| {
selcx.infcx.register_region_outlives_constraint(pred, &dummy_cause);
});
}
ty::PredicateKind::Clause(ty::ClauseKind::TypeOutlives(binder)) => {
let binder = bound_predicate.rebind(binder);
@ -735,14 +737,14 @@ impl<'tcx> AutoTraitFinder<'tcx> {
binder.map_bound_ref(|pred| pred.0).no_bound_vars(),
) {
(None, Some(t_a)) => {
selcx.infcx.register_region_obligation_with_cause(
selcx.infcx.register_type_outlives_constraint(
t_a,
selcx.infcx.tcx.lifetimes.re_static,
&dummy_cause,
);
}
(Some(ty::OutlivesPredicate(t_a, r_b)), _) => {
selcx.infcx.register_region_obligation_with_cause(
selcx.infcx.register_type_outlives_constraint(
t_a,
r_b,
&dummy_cause,

4
compiler/rustc_trait_selection/src/traits/fulfill.rs
View file

@ -428,7 +428,7 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
ty::PredicateKind::Clause(ty::ClauseKind::RegionOutlives(data)) => {
if infcx.considering_regions {
infcx.region_outlives_predicate(&obligation.cause, Binder::dummy(data));
infcx.register_region_outlives_constraint(data, &obligation.cause);
}
ProcessResult::Changed(Default::default())
@ -439,7 +439,7 @@ impl<'a, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'tcx> {
r_b,
))) => {
if infcx.considering_regions {
infcx.register_region_obligation_with_cause(t_a, r_b, &obligation.cause);
infcx.register_type_outlives_constraint(t_a, r_b, &obligation.cause);
}
ProcessResult::Changed(Default::default())
}

Some files were not shown because too many files have changed in this diff Show more