user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Auto merge of #3935 - RalfJung:rustup, r=RalfJung

Browse source 
Rustup
This commit is contained in:
bors 2024-10-03 06:20:27 +00:00
commit bf71d65980
950 changed files with 12185 additions and 6996 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.github/pull_request_template.md vendored
View file

@ -7,6 +7,6 @@ tracking issue or there are none, feel free to ignore this.
This PR will get automatically assigned to a reviewer. In case you would like
a specific user to review your work, you can assign it to them by using
r​? <reviewer name>
r\? <reviewer name> (with the `\` removed)
-->
<!-- homu-ignore:end -->

10
.github/workflows/ci.yml vendored
View file

@ -212,6 +212,16 @@ jobs:
# erroring about invalid credentials instead.
if: github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1'
- name: upload job metrics to DataDog
if: needs.calculate_matrix.outputs.run_type != 'pr'
env:
DATADOG_SITE: datadoghq.com
DATADOG_API_KEY: ${{ secrets.DATADOG_API_KEY }}
DD_GITHUB_JOB_NAME: ${{ matrix.name }}
run: |
npm install -g @datadog/datadog-ci@^2.x.x
python3 src/ci/scripts/upload-build-metrics.py build/cpu-usage.csv
# This job isused to tell bors the final status of the build, as there is no practical way to detect
# when a workflow is successful listening to webhooks only in our current bors implementation (homu).
outcome:

6
.github/workflows/dependencies.yml vendored
View file

@ -61,9 +61,11 @@ jobs:
rustup toolchain install --no-self-update --profile minimal $TOOLCHAIN
rustup default $TOOLCHAIN
- name: cargo update
- name: cargo update compiler & tools
# Remove first line that always just says "Updating crates.io index"
run: cargo update 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
run: |
echo -e "\ncompiler & tools dependencies:" >> cargo_update.log
cargo update 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
- name: cargo update library
run: |
echo -e "\nlibrary dependencies:" >> cargo_update.log

1
.gitignore vendored
View file

@ -25,6 +25,7 @@ Session.vim
.favorites.json
.settings/
.vs/
.dir-locals.el
## Tool
.valgrindrc

565
Cargo.lock
View file

File diff suppressed because it is too large Load diff

10
compiler/rustc_attr/src/builtin.rs
View file

@ -80,6 +80,10 @@ impl Stability {
pub fn is_stable(&self) -> bool {
self.level.is_stable()
}
pub fn stable_since(&self) -> Option<StableSince> {
self.level.stable_since()
}
}
/// Represents the `#[rustc_const_unstable]` and `#[rustc_const_stable]` attributes.
@ -170,6 +174,12 @@ impl StabilityLevel {
pub fn is_stable(&self) -> bool {
matches!(self, StabilityLevel::Stable { .. })
}
pub fn stable_since(&self) -> Option<StableSince> {
match *self {
StabilityLevel::Stable { since, .. } => Some(since),
StabilityLevel::Unstable { .. } => None,
}
}
}
#[derive(Encodable, Decodable, PartialEq, Copy, Clone, Debug, Eq, Hash)]

12
compiler/rustc_borrowck/src/dataflow.rs
View file

@ -1,5 +1,3 @@
use std::fmt;
use rustc_data_structures::fx::FxIndexMap;
use rustc_data_structures::graph;
use rustc_index::bit_set::BitSet;
@ -425,10 +423,6 @@ impl<'a, 'tcx> Borrows<'a, 'tcx> {
Borrows { tcx, body, borrow_set, borrows_out_of_scope_at_location }
}
pub fn location(&self, idx: BorrowIndex) -> &Location {
&self.borrow_set[idx].reserve_location
}
/// Add all borrows to the kill set, if those borrows are out of scope at `location`.
/// That means they went out of a nonlexical scope
fn kill_loans_out_of_scope_at_location(
@ -615,8 +609,4 @@ impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
}
}
impl DebugWithContext<Borrows<'_, '_>> for BorrowIndex {
fn fmt_with(&self, ctxt: &Borrows<'_, '_>, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{:?}", ctxt.location(*self))
}
}
impl<C> DebugWithContext<C> for BorrowIndex {}

2
compiler/rustc_borrowck/src/type_check/mod.rs
View file

@ -2437,7 +2437,7 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
debug!(?src_tty, ?dst_tty, ?src_obj, ?dst_obj);
self.eq_types(
self.sub_types(
src_obj,
dst_obj,
location.to_locations(),

10
compiler/rustc_builtin_macros/src/asm.rs
View file

@ -507,6 +507,7 @@ fn expand_preparsed_asm(
let msg = "asm template must be a string literal";
let template_sp = template_expr.span;
let template_is_mac_call = matches!(template_expr.kind, ast::ExprKind::MacCall(_));
let (template_str, template_style, template_span) = {
let ExpandResult::Ready(mac) = expr_to_spanned_string(ecx, template_expr, msg) else {
return ExpandResult::Retry(());
@ -596,7 +597,14 @@ fn expand_preparsed_asm(
if !parser.errors.is_empty() {
let err = parser.errors.remove(0);
let err_sp = template_span.from_inner(InnerSpan::new(err.span.start, err.span.end));
let err_sp = if template_is_mac_call {
// If the template is a macro call we can't reliably point to the error's
// span so just use the template's span as the error span (fixes #129503)
template_span
} else {
template_span.from_inner(InnerSpan::new(err.span.start, err.span.end))
};
let msg = format!("invalid asm template string: {}", err.description);
let mut e = ecx.dcx().struct_span_err(err_sp, msg);
e.span_label(err_sp, err.label + " in asm template string");

2
compiler/rustc_builtin_macros/src/cfg.rs
View file

@ -43,7 +43,7 @@ fn parse_cfg<'a>(cx: &ExtCtxt<'a>, span: Span, tts: TokenStream) -> PResult<'a,
return Err(cx.dcx().create_err(errors::RequiresCfgPattern { span }));
}
let cfg = p.parse_meta_item(AllowLeadingUnsafe::Yes)?;
let cfg = p.parse_meta_item(AllowLeadingUnsafe::No)?;
let _ = p.eat(&token::Comma);

2
compiler/rustc_builtin_macros/src/deriving/smart_ptr.rs
View file

@ -310,7 +310,7 @@ pub(crate) fn expand_deriving_smart_ptr(
// Add the impl blocks for `DispatchFromDyn` and `CoerceUnsized`.
let gen_args = vec![GenericArg::Type(alt_self_type.clone())];
add_impl_block(impl_generics.clone(), sym::DispatchFromDyn, gen_args.clone());
add_impl_block(impl_generics.clone(), sym::CoerceUnsized, gen_args.clone());
add_impl_block(impl_generics.clone(), sym::CoerceUnsized, gen_args);
}
fn contains_maybe_sized_bound_on_pointee(predicates: &[WherePredicate], pointee: Symbol) -> bool {

13
compiler/rustc_codegen_cranelift/patches/0027-stdlib-128bit-atomic-operations.patch
View file

@ -82,19 +82,6 @@ index d9de37e..8293fce 100644
#[cfg(target_has_atomic_load_store = "ptr")]
macro_rules! atomic_int_ptr_sized {
( $($target_pointer_width:literal $align:literal)* ) => { $(
diff --git a/library/core/src/cell.rs b/library/core/src/cell.rs
index 58b9ba4..91bbd0a 100644
--- a/library/core/src/cell.rs
+++ b/library/core/src/cell.rs
@@ -2246,8 +2246,6 @@ unsafe_cell_primitive_into_inner! {
u32 "32"
i64 "64"
u64 "64"
- i128 "128"
- u128 "128"
isize "ptr"
usize "ptr"
}
--
2.26.2.7.g19db9cfb68

17
compiler/rustc_codegen_cranelift/src/unsize.rs
View file

@ -2,6 +2,7 @@
//!
//! [`PointerCoercion::Unsize`]: `rustc_middle::ty::adjustment::PointerCoercion::Unsize`
use rustc_codegen_ssa::base::validate_trivial_unsize;
use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
use crate::base::codegen_panic_nounwind;
@ -34,20 +35,8 @@ pub(crate) fn unsized_info<'tcx>(
let old_info =
old_info.expect("unsized_info: missing old info for trait upcasting coercion");
if data_a.principal_def_id() == data_b.principal_def_id() {
// Codegen takes advantage of the additional assumption, where if the
// principal trait def id of what's being casted doesn't change,
// then we don't need to adjust the vtable at all. This
// corresponds to the fact that `dyn Tr<A>: Unsize<dyn Tr<B>>`
// requires that `A = B`; we don't allow *upcasting* objects
// between the same trait with different args. If we, for
// some reason, were to relax the `Unsize` trait, it could become
// unsound, so let's assert here that the trait refs are *equal*.
//
// We can use `assert_eq` because the binders should have been anonymized,
// and because higher-ranked equality now requires the binders are equal.
debug_assert_eq!(
data_a.principal(),
data_b.principal(),
debug_assert!(
validate_trivial_unsize(fx.tcx, data_a, data_b),
"NOP unsize vtable changed principal trait ref: {data_a} -> {data_b}"
);
return old_info;

6
compiler/rustc_codegen_gcc/.github/workflows/ci.yml vendored
View file

@ -36,7 +36,7 @@ jobs:
]
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
# `rustup show` installs from rust-toolchain.toml
- name: Setup rust toolchain
@ -113,13 +113,13 @@ jobs:
duplicates:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- run: python tools/check_intrinsics_duplicates.py
build_system:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- name: Test build system
run: |
cd build_system

2
compiler/rustc_codegen_gcc/.github/workflows/failures.yml vendored
View file

@ -31,7 +31,7 @@ jobs:
env_extra: "TEST_FLAGS='-Cpanic=abort -Zpanic-abort-tests' GCC_EXEC_PREFIX=/usr/lib/gcc/"
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
# `rustup show` installs from rust-toolchain.toml
- name: Setup rust toolchain

2
compiler/rustc_codegen_gcc/.github/workflows/gcc12.yml vendored
View file

@ -33,7 +33,7 @@ jobs:
]
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
# `rustup show` installs from rust-toolchain.toml
- name: Setup rust toolchain

8
compiler/rustc_codegen_gcc/.github/workflows/m68k.yml vendored
View file

@ -36,7 +36,7 @@ jobs:
]
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
# `rustup show` installs from rust-toolchain.toml
- name: Setup rust toolchain
@ -85,14 +85,14 @@ jobs:
- name: Build sample project with target defined as JSON spec
run: |
./y.sh prepare --only-libcore --cross
./y.sh build --sysroot --target-triple m68k-unknown-linux-gnu --target ${{ github.workspace }}/target_specs/m68k-unknown-linux-gnu.json
./y.sh build --sysroot --features compiler_builtins/no-f16-f128 --target-triple m68k-unknown-linux-gnu --target ${{ github.workspace }}/target_specs/m68k-unknown-linux-gnu.json
./y.sh cargo build --manifest-path=./tests/hello-world/Cargo.toml --target ${{ github.workspace }}/target_specs/m68k-unknown-linux-gnu.json
./y.sh clean all
- name: Build
run: |
./y.sh prepare --only-libcore --cross
./y.sh build --sysroot --target-triple m68k-unknown-linux-gnu
./y.sh build --sysroot --features compiler_builtins/no-f16-f128 --target-triple m68k-unknown-linux-gnu
CG_GCC_TEST_TARGET=m68k-unknown-linux-gnu cargo test
./y.sh clean all
@ -107,4 +107,4 @@ jobs:
- name: Run tests
run: |
./y.sh test --release --clean --build-sysroot ${{ matrix.commands }}
./y.sh test --release --clean --build-sysroot --sysroot-features compiler_builtins/no-f16-f128 ${{ matrix.commands }}

2
compiler/rustc_codegen_gcc/.github/workflows/release.yml vendored
View file

@ -24,7 +24,7 @@ jobs:
]
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
# `rustup show` installs from rust-toolchain.toml
- name: Setup rust toolchain

14
compiler/rustc_codegen_gcc/.github/workflows/stdarch.yml vendored
View file

@ -13,7 +13,7 @@ env:
jobs:
build:
runs-on: ubuntu-latest
runs-on: ubuntu-24.04
strategy:
fail-fast: false
@ -24,7 +24,7 @@ jobs:
]
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
# `rustup show` installs from rust-toolchain.toml
- name: Setup rust toolchain
@ -36,6 +36,13 @@ jobs:
- name: Install packages
run: sudo apt-get install ninja-build ripgrep
# TODO: remove when we have binutils version 2.43 in the repo.
- name: Install more recent binutils
run: |
echo "deb http://archive.ubuntu.com/ubuntu oracular main universe" | sudo tee /etc/apt/sources.list.d/oracular-copies.list
sudo apt-get update
sudo apt-get install binutils
- name: Install Intel Software Development Emulator
if: ${{ matrix.cargo_runner }}
run: |
@ -96,4 +103,5 @@ jobs:
run: |
# FIXME: these tests fail when the sysroot is compiled with LTO because of a missing symbol in proc-macro.
# TODO: remove --skip test_mm512_stream_ps when stdarch is updated in rustc.
STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features" ./y.sh cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml -- --skip rtm --skip tbm --skip sse4a --skip test_mm512_stream_ps
# TODO: remove --skip test_tile_ when it's implemented.
STDARCH_TEST_EVERYTHING=1 CHANNEL=release CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="${{ matrix.cargo_runner }}" TARGET=x86_64-unknown-linux-gnu CG_RUSTFLAGS="-Ainternal_features --cfg stdarch_intel_sde" ./y.sh cargo test --manifest-path build/build_sysroot/sysroot_src/library/stdarch/Cargo.toml -- --skip rtm --skip tbm --skip sse4a --skip test_mm512_stream_ps --skip test_tile_

8
compiler/rustc_codegen_gcc/Cargo.lock
View file

@ -28,18 +28,18 @@ dependencies = [
[[package]]
name = "gccjit"
version = "2.1.0"
version = "2.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62e0ba949ebee07c5cc21f02cb48f28f2c8db7fcbc15fdc5120476a6c43b4636"
checksum = "4bb376e98c82d9284c3a17fc1d6bf9bc921055418950238d7a553c27a7e1f6ab"
dependencies = [
"gccjit_sys",
]
[[package]]
name = "gccjit_sys"
version = "0.2.0"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5bbf85e12c2593772329a9d4e8310271f6706e6045ce4f41b041dd34fba6603"
checksum = "93b4b1be553b5df790bf25ca2a1d6add81727dc29f8d5c8742468ed306d621d1"
dependencies = [
"libc",
]

4
compiler/rustc_codegen_gcc/Cargo.toml
View file

@ -22,7 +22,9 @@ master = ["gccjit/master"]
default = ["master"]
[dependencies]
gccjit = "2.1"
gccjit = "2.2"
#gccjit = { git = "https://github.com/rust-lang/gccjit.rs" }
# Local copy.
#gccjit = { path = "../gccjit.rs" }

99
compiler/rustc_codegen_gcc/build_system/build_sysroot/Cargo.lock
View file

@ -4,12 +4,12 @@ version = 3
[[package]]
name = "addr2line"
version = "0.21.0"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb"
checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678"
dependencies = [
"compiler_builtins",
"gimli",
"gimli 0.29.0",
"rustc-std-workspace-alloc",
"rustc-std-workspace-core",
]
@ -52,7 +52,7 @@ dependencies = [
name = "compiler_builtins"
version = "0.1.118"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f11973008a8cf741fe6d22f339eba21fd0ca81e2760a769ba8243ed6c21edd7e"
checksum = "92afe7344b64cccf3662ca26d5d1c0828ab826f04206b97d856e3625e390e4b5"
dependencies = [
"rustc-std-workspace-core",
]
@ -97,9 +97,20 @@ dependencies = [
[[package]]
name = "gimli"
version = "0.28.1"
version = "0.29.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253"
checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-alloc",
"rustc-std-workspace-core",
]
[[package]]
name = "gimli"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2e1d97fbe9722ba9bbd0c97051c2956e726562b61f86a25a4360398a40edfc9"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-alloc",
@ -120,9 +131,9 @@ dependencies = [
[[package]]
name = "hermit-abi"
version = "0.3.9"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
checksum = "fbf6a919d6cf397374f7dfeeea91d974c7c0a7221d0d0f4f20d859d329e53fcc"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-alloc",
@ -131,18 +142,18 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.153"
version = "0.2.155"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
dependencies = [
"rustc-std-workspace-core",
]
[[package]]
name = "memchr"
version = "2.7.2"
version = "2.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d"
checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-core",
@ -150,9 +161,9 @@ dependencies = [
[[package]]
name = "miniz_oxide"
version = "0.7.2"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7"
checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08"
dependencies = [
"adler",
"compiler_builtins",
@ -162,9 +173,9 @@ dependencies = [
[[package]]
name = "object"
version = "0.32.2"
version = "0.36.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441"
checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9"
dependencies = [
"compiler_builtins",
"memchr",
@ -205,9 +216,9 @@ dependencies = [
[[package]]
name = "r-efi"
version = "4.4.0"
version = "4.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c47196f636c4cc0634b73b0405323d177753c2e15e866952c64ea22902567a34"
checksum = "e9e935efc5854715dfc0a4c9ef18dc69dee0ec3bf9cc3ab740db831c0fdd86a3"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-core",
@ -226,9 +237,9 @@ dependencies = [
[[package]]
name = "rustc-demangle"
version = "0.1.23"
version = "0.1.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-core",
@ -310,16 +321,14 @@ dependencies = [
"core",
"getopts",
"libc",
"panic_abort",
"panic_unwind",
"std",
]
[[package]]
name = "unicode-width"
version = "0.1.12"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68f5e5f3158ecfd4b8ff6fe086db7c8467a2dfdac97fe420f2b7c4aa97af66d6"
checksum = "0336d538f7abc86d282a4189614dfaa90810dfc2c6f6427eaf88e16311dd225d"
dependencies = [
"compiler_builtins",
"rustc-std-workspace-core",
@ -339,12 +348,12 @@ dependencies = [
[[package]]
name = "unwinding"
version = "0.2.1"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37a19a21a537f635c16c7576f22d0f2f7d63353c1337ad4ce0d8001c7952a25b"
checksum = "dc55842d0db6329a669d55a623c674b02d677b16bfb2d24857d4089d41eba882"
dependencies = [
"compiler_builtins",
"gimli",
"gimli 0.30.0",
"rustc-std-workspace-core",
]
@ -370,9 +379,9 @@ dependencies = [
[[package]]
name = "windows-targets"
version = "0.52.5"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
@ -386,48 +395,48 @@ dependencies = [
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.5"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.5"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_i686_gnu"
version = "0.52.5"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.5"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_msvc"
version = "0.52.5"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.5"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.5"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.5"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"

16
compiler/rustc_codegen_gcc/build_system/build_sysroot/Cargo.toml
View file

@ -17,6 +17,22 @@ rustc-std-workspace-core = { path = "./sysroot_src/library/rustc-std-workspace-c
rustc-std-workspace-alloc = { path = "./sysroot_src/library/rustc-std-workspace-alloc" }
rustc-std-workspace-std = { path = "./sysroot_src/library/rustc-std-workspace-std" }
# For compiler-builtins we always use a high number of codegen units.
# The goal here is to place every single intrinsic into its own object
# file to avoid symbol clashes with the system libgcc if possible. Note
# that this number doesn't actually produce this many object files, we
# just don't create more than this number of object files.
#
# It's a bit of a bummer that we have to pass this here, unfortunately.
# Ideally this would be specified through an env var to Cargo so Cargo
# knows how many CGUs are for this specific crate, but for now
# per-crate configuration isn't specifiable in the environment.
[profile.dev.package.compiler_builtins]
codegen-units = 10000
[profile.release.package.compiler_builtins]
codegen-units = 10000
[profile.release]
debug = "limited"
#lto = "fat" # TODO(antoyo): re-enable when the failing LTO tests regarding proc-macros are fixed.

24
compiler/rustc_codegen_gcc/build_system/src/build.rs
View file

@ -24,16 +24,6 @@ impl BuildArg {
while let Some(arg) = args.next() {
match arg.as_str() {
"--features" => {
if let Some(arg) = args.next() {
build_arg.flags.push("--features".to_string());
build_arg.flags.push(arg.as_str().into());
} else {
return Err(
"Expected a value after `--features`, found nothing".to_string()
);
}
}
"--sysroot" => {
build_arg.build_sysroot = true;
}
@ -56,7 +46,6 @@ impl BuildArg {
r#"
`build` command help:
--features [arg] : Add a new feature [arg]
--sysroot : Build with sysroot"#
);
ConfigInfo::show_usage();
@ -142,14 +131,11 @@ pub fn build_sysroot(env: &HashMap<String, String>, config: &ConfigInfo) -> Resu
rustflags.push_str(" -Csymbol-mangling-version=v0");
}
let mut args: Vec<&dyn AsRef<OsStr>> = vec![
&"cargo",
&"build",
&"--target",
&config.target,
&"--features",
&"compiler-builtins-no-f16-f128",
];
let mut args: Vec<&dyn AsRef<OsStr>> = vec![&"cargo", &"build", &"--target", &config.target];
for feature in &config.features {
args.push(&"--features");
args.push(feature);
}
if config.no_default_features {
rustflags.push_str(" -Csymbol-mangling-version=v0");

11
compiler/rustc_codegen_gcc/build_system/src/config.rs
View file

@ -98,7 +98,7 @@ impl ConfigFile {
}
}
#[derive(Default, Debug)]
#[derive(Default, Debug, Clone)]
pub struct ConfigInfo {
pub target: String,
pub target_triple: String,
@ -123,6 +123,7 @@ pub struct ConfigInfo {
pub no_download: bool,
pub no_default_features: bool,
pub backend: Option<String>,
pub features: Vec<String>,
}
impl ConfigInfo {
@ -133,6 +134,13 @@ impl ConfigInfo {
args: &mut impl Iterator<Item = String>,
) -> Result<bool, String> {
match arg {
"--features" => {
if let Some(arg) = args.next() {
self.features.push(arg);
} else {
return Err("Expected a value after `--features`, found nothing".to_string());
}
}
"--target" => {
if let Some(arg) = args.next() {
self.target = arg;
@ -443,6 +451,7 @@ impl ConfigInfo {
pub fn show_usage() {
println!(
"\
--features [arg] : Add a new feature [arg]
--target-triple [arg] : Set the target triple to [arg]
--target [arg] : Set the target to [arg]
--out-dir : Location where the files will be generated

16
compiler/rustc_codegen_gcc/build_system/src/test.rs
View file

@ -92,6 +92,7 @@ struct TestArg {
current_part: Option<usize>,
sysroot_panic_abort: bool,
config_info: ConfigInfo,
sysroot_features: Vec<String>,
}
impl TestArg {
@ -127,6 +128,14 @@ impl TestArg {
"--sysroot-panic-abort" => {
test_arg.sysroot_panic_abort = true;
}
"--sysroot-features" => match args.next() {
Some(feature) if !feature.is_empty() => {
test_arg.sysroot_features.push(feature);
}
_ => {
return Err(format!("Expected an argument after `{}`, found nothing", arg));
}
},
"--help" => {
show_usage();
return Ok(None);
@ -250,7 +259,9 @@ fn mini_tests(env: &Env, args: &TestArg) -> Result<(), String> {
fn build_sysroot(env: &Env, args: &TestArg) -> Result<(), String> {
// FIXME: create a function "display_if_not_quiet" or something along the line.
println!("[BUILD] sysroot");
build::build_sysroot(env, &args.config_info)?;
let mut config = args.config_info.clone();
config.features.extend(args.sysroot_features.iter().cloned());
build::build_sysroot(env, &config)?;
Ok(())
}
@ -626,7 +637,8 @@ fn test_projects(env: &Env, args: &TestArg) -> Result<(), String> {
"https://github.com/BurntSushi/memchr",
"https://github.com/dtolnay/itoa",
"https://github.com/rust-lang/cfg-if",
"https://github.com/rust-lang-nursery/lazy-static.rs",
//"https://github.com/rust-lang-nursery/lazy-static.rs", // TODO: re-enable when the
//failing test is fixed upstream.
//"https://github.com/marshallpierce/rust-base64", // FIXME: one test is OOM-killed.
// TODO: ignore the base64 test that is OOM-killed.
"https://github.com/time-rs/time",

3
compiler/rustc_codegen_gcc/doc/debugging-gcc-lto.md
View file

@ -1,3 +0,0 @@
# How to debug GCC LTO
Run do the command with `-v -save-temps` and then extract the `lto1` line from the output and run that under the debugger.

38
compiler/rustc_codegen_gcc/doc/debugging.md Normal file
View file

@ -0,0 +1,38 @@
# Debugging
## How to debug GCC LTO
Run do the command with `-v -save-temps` and then extract the `lto1` line from the output and run that under the debugger.
## How to debug stdarch tests that cannot be ran locally
First, run the tests normally:
----
cd build/build_sysroot/sysroot_src/library/stdarch/
STDARCH_TEST_EVERYTHING=1 CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUNNER="sde -future -rtm_mode full --" TARGET=x86_64-unknown-linux-gnu ../../../../../y.sh cargo test
----
It will show the command it ran, something like this:
----
process didn't exit successfully: `sde -future -rtm_mode full -- /home/user/projects/rustc_codegen_gcc/build/build_sysroot/sysroot_src/library/stdarch/target/debug/deps/core_arch-fd2d75f89baae5c6` (signal: 11, SIGSEGV: invalid memory reference)
----
Then add the `-debug` flag to it:
----
sde -debug -future -rtm_mode full -- /home/user/projects/rustc_codegen_gcc/build/build_sysroot/sysroot_src/library/stdarch/target/debug/deps/core_arch-fd2d75f89baae5c6
----
To see the symbols in `gdb`, specify the executable in your command:
----
gdb /home/user/projects/rustc_codegen_gcc/build/build_sysroot/sysroot_src/library/stdarch/target/debug/deps/core_arch-fd2d75f89baae5c6
----
and then write the `gdb` command that `sde` tells you to use, something like:
----
target remote :51299
----

2
compiler/rustc_codegen_gcc/libgccjit.version
View file

@ -1 +1 @@
341be3b7d7ac6976cfed8ed59da3573c040d0776
e744a9459d33864067214741daf5c5bc2a7b88c6

37
compiler/rustc_codegen_gcc/patches/0022-core-Disable-not-compiling-tests.patch
View file

@ -1,26 +1,24 @@
From f6befc4bb51d84f5f1cf35938a168c953d421350 Mon Sep 17 00:00:00 2001
From: bjorn3 <bjorn3@users.noreply.github.com>
Date: Sun, 24 Nov 2019 15:10:23 +0100
From 18793c6109890493ceb3ff36549849a36e3d8022 Mon Sep 17 00:00:00 2001
From: None <none@example.com>
Date: Sun, 1 Sep 2024 11:42:17 -0400
Subject: [PATCH] [core] Disable not compiling tests
---
library/core/tests/Cargo.toml | 8 ++++++++
library/core/tests/num/flt2dec/mod.rs | 1 -
library/core/tests/num/int_macros.rs | 2 ++
library/core/tests/num/uint_macros.rs | 2 ++
library/core/tests/ptr.rs | 2 ++
library/core/tests/slice.rs | 2 ++
6 files changed, 16 insertions(+), 1 deletion(-)
library/core/tests/Cargo.toml | 14 ++++++++++++++
library/core/tests/lib.rs | 1 +
2 files changed, 15 insertions(+)
create mode 100644 library/core/tests/Cargo.toml
diff --git a/library/core/tests/Cargo.toml b/library/core/tests/Cargo.toml
new file mode 100644
index 0000000..46fd999
index 0000000..ca326ac
--- /dev/null
+++ b/library/core/tests/Cargo.toml
@@ -0,0 +1,12 @@
@@ -0,0 +1,14 @@
+[workspace]
+
+[package]
+name = "core"
+name = "coretests"
+version = "0.0.0"
+edition = "2021"
+
@ -32,11 +30,14 @@ index 0000000..46fd999
+rand = { version = "0.8.5", default-features = false }
+rand_xorshift = { version = "0.3.0", default-features = false }
diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs
index 42a26ae..5ac1042 100644
index 1e336bf..5800ebb 100644
--- a/library/core/tests/lib.rs
+++ b/library/core/tests/lib.rs
@@ -1,3 +1,4 @@
@@ -1,4 +1,5 @@
// tidy-alphabetical-start
+#![cfg(test)]
#![feature(alloc_layout_extra)]
#![feature(array_chunks)]
#![feature(array_ptr_get)]
#![cfg_attr(bootstrap, feature(offset_of_nested))]
#![cfg_attr(target_has_atomic = "128", feature(integer_atomics))]
#![cfg_attr(test, feature(cfg_match))]
--
2.46.0

12
compiler/rustc_codegen_gcc/patches/libgccjit12/0001-core-Disable-portable-simd-test.patch
View file

@ -11,15 +11,15 @@ diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs
index b71786c..cf484d5 100644
--- a/library/core/tests/lib.rs
+++ b/library/core/tests/lib.rs
@@ -95,7 +95,6 @@
#![feature(never_type)]
#![feature(unwrap_infallible)]
@@ -87,7 +87,6 @@
#![feature(numfmt)]
#![feature(pattern)]
#![feature(pointer_is_aligned_to)]
-#![feature(portable_simd)]
#![feature(ptr_metadata)]
#![feature(unsized_tuple_coercion)]
#![feature(const_option)]
@@ -157,7 +156,6 @@ mod pin;
#![feature(slice_from_ptr_range)]
#![feature(slice_internals)]
@@ -155,7 +154,6 @@ mod pin;
mod pin_macro;
mod ptr;
mod result;

2
compiler/rustc_codegen_gcc/rust-toolchain
View file

@ -1,3 +1,3 @@
[toolchain]
channel = "nightly-2024-07-02"
channel = "nightly-2024-08-11"
components = ["rust-src", "rustc-dev", "llvm-tools-preview"]

13
compiler/rustc_codegen_gcc/src/allocator.rs
View file

@ -104,10 +104,17 @@ fn create_wrapper_function(
false,
);
if tcx.sess.default_hidden_visibility() {
#[cfg(feature = "master")]
func.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden));
#[cfg(feature = "master")]
match tcx.sess.default_visibility() {
rustc_target::spec::SymbolVisibility::Hidden => {
func.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Hidden))
}
rustc_target::spec::SymbolVisibility::Protected => {
func.add_attribute(FnAttribute::Visibility(gccjit::Visibility::Protected))
}
rustc_target::spec::SymbolVisibility::Interposable => {}
}
if tcx.sess.must_emit_unwind_tables() {
// TODO(antoyo): emit unwind tables.
}

8
compiler/rustc_codegen_gcc/src/asm.rs
View file

@ -682,6 +682,11 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r",
InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg_addr) => "a",
InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f",
InlineAsmRegClass::S390x(
S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg,
) => {
unreachable!("clobber-only")
}
InlineAsmRegClass::Err => unreachable!(),
},
};
@ -757,6 +762,9 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr,
) => cx.type_i32(),
InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
InlineAsmRegClass::S390x(S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg) => {
unreachable!("clobber-only")
}
InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
InlineAsmRegClass::M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),

13
compiler/rustc_codegen_gcc/src/base.rs
View file

@ -128,8 +128,19 @@ pub fn compile_codegen_unit(
// NOTE: Rust relies on LLVM doing wrapping on overflow.
context.add_command_line_option("-fwrapv");
if let Some(model) = tcx.sess.code_model() {
use rustc_target::spec::CodeModel;
context.add_command_line_option(match model {
CodeModel::Tiny => "-mcmodel=tiny",
CodeModel::Small => "-mcmodel=small",
CodeModel::Kernel => "-mcmodel=kernel",
CodeModel::Medium => "-mcmodel=medium",
CodeModel::Large => "-mcmodel=large",
});
}
if tcx.sess.relocation_model() == rustc_target::spec::RelocModel::Static {
context.add_command_line_option("-mcmodel=kernel");
context.add_command_line_option("-fno-pie");
}

54
compiler/rustc_codegen_gcc/src/builder.rs
View file

@ -39,9 +39,6 @@ use crate::type_of::LayoutGccExt;
// TODO(antoyo)
type Funclet = ();
// TODO(antoyo): remove this variable.
static mut RETURN_VALUE_COUNT: usize = 0;
enum ExtremumOperation {
Max,
Min,
@ -50,13 +47,18 @@ enum ExtremumOperation {
pub struct Builder<'a: 'gcc, 'gcc, 'tcx> {
pub cx: &'a CodegenCx<'gcc, 'tcx>,
pub block: Block<'gcc>,
stack_var_count: Cell<usize>,
pub location: Option<Location<'gcc>>,
value_counter: Cell<u64>,
}
impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
fn with_cx(cx: &'a CodegenCx<'gcc, 'tcx>, block: Block<'gcc>) -> Self {
Builder { cx, block, stack_var_count: Cell::new(0), location: None }
Builder { cx, block, location: None, value_counter: Cell::new(0) }
}
fn next_value_counter(&self) -> u64 {
self.value_counter.set(self.value_counter.get() + 1);
self.value_counter.get()
}
fn atomic_extremum(
@ -138,7 +140,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
) -> RValue<'gcc> {
let size = get_maybe_pointer_size(src);
let compare_exchange =
self.context.get_builtin_function(&format!("__atomic_compare_exchange_{}", size));
self.context.get_builtin_function(format!("__atomic_compare_exchange_{}", size));
let order = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc());
let failure_order = self.context.new_rvalue_from_int(self.i32_type, failure_order.to_gcc());
let weak = self.context.new_rvalue_from_int(self.bool_type, weak as i32);
@ -270,10 +272,12 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
actual_val.dereference(self.location).to_rvalue()
}
} else {
// FIXME: this condition seems wrong: it will pass when both types are not
// a vector.
assert!(
(!expected_ty.is_vector() || actual_ty.is_vector())
&& (expected_ty.is_vector() || !actual_ty.is_vector()),
"{:?} ({}) -> {:?} ({}), index: {:?}[{}]",
"{:?} (is vector: {}) -> {:?} (is vector: {}), Function: {:?}[{}]",
actual_ty,
actual_ty.is_vector(),
expected_ty,
@ -283,6 +287,8 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
);
// TODO(antoyo): perhaps use __builtin_convertvector for vector casting.
// TODO: remove bitcast now that vector types can be compared?
// ==> We use bitcast to avoid having to do many manual casts from e.g. __m256i to __v32qi (in
// the case of _mm256_aesenc_epi128).
self.bitcast(actual_val, expected_ty)
}
} else {
@ -325,11 +331,10 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
let void_type = self.context.new_type::<()>();
let current_func = self.block.get_function();
if return_type != void_type {
unsafe { RETURN_VALUE_COUNT += 1 };
let result = current_func.new_local(
self.location,
return_type,
&format!("returnValue{}", unsafe { RETURN_VALUE_COUNT }),
format!("returnValue{}", self.next_value_counter()),
);
self.block.add_assignment(
self.location,
@ -341,7 +346,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
self.block
.add_eval(self.location, self.cx.context.new_call(self.location, func, &args));
// Return dummy value when not having return value.
self.context.new_rvalue_from_long(self.isize_type, 0)
self.context.new_rvalue_zero(self.isize_type)
}
}
@ -367,6 +372,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
let args = {
let function_address_names = self.function_address_names.borrow();
let original_function_name = function_address_names.get(&func_ptr);
func_ptr = llvm::adjust_function(self.context, &func_name, func_ptr, args);
llvm::adjust_intrinsic_arguments(
self,
gcc_func,
@ -385,7 +391,6 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
let current_func = self.block.get_function();
if return_type != void_type {
unsafe { RETURN_VALUE_COUNT += 1 };
let return_value = self.cx.context.new_call_through_ptr(self.location, func_ptr, &args);
let return_value = llvm::adjust_intrinsic_return_value(
self,
@ -398,7 +403,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
let result = current_func.new_local(
self.location,
return_value.get_type(),
&format!("ptrReturnValue{}", unsafe { RETURN_VALUE_COUNT }),
format!("ptrReturnValue{}", self.next_value_counter()),
);
self.block.add_assignment(self.location, result, return_value);
result.to_rvalue()
@ -422,17 +427,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
self.cx.context.new_call_through_ptr(self.location, func_ptr, &args),
);
// Return dummy value when not having return value.
let result = current_func.new_local(
self.location,
self.isize_type,
"dummyValueThatShouldNeverBeUsed",
);
self.block.add_assignment(
self.location,
result,
self.context.new_rvalue_from_long(self.isize_type, 0),
);
result.to_rvalue()
self.context.new_rvalue_zero(self.isize_type)
}
}
@ -447,11 +442,10 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
let return_type = self.context.new_type::<bool>();
let current_func = self.block.get_function();
// TODO(antoyo): return the new_call() directly? Since the overflow function has no side-effects.
unsafe { RETURN_VALUE_COUNT += 1 };
let result = current_func.new_local(
self.location,
return_type,
&format!("overflowReturnValue{}", unsafe { RETURN_VALUE_COUNT }),
format!("overflowReturnValue{}", self.next_value_counter()),
);
self.block.add_assignment(
self.location,
@ -930,9 +924,8 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
fn alloca(&mut self, size: Size, align: Align) -> RValue<'gcc> {
let ty = self.cx.type_array(self.cx.type_i8(), size.bytes()).get_aligned(align.bytes());
// TODO(antoyo): It might be better to return a LValue, but fixing the rustc API is non-trivial.
self.stack_var_count.set(self.stack_var_count.get() + 1);
self.current_func()
.new_local(self.location, ty, &format!("stack_var_{}", self.stack_var_count.get()))
.new_local(self.location, ty, format!("stack_var_{}", self.next_value_counter()))
.get_address(self.location)
}
@ -955,11 +948,10 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
};
let ptr = self.context.new_cast(self.location, ptr, aligned_type.make_pointer());
let deref = ptr.dereference(self.location).to_rvalue();
unsafe { RETURN_VALUE_COUNT += 1 };
let loaded_value = function.new_local(
self.location,
aligned_type,
&format!("loadedValue{}", unsafe { RETURN_VALUE_COUNT }),
format!("loadedValue{}", self.next_value_counter()),
);
block.add_assignment(self.location, loaded_value, deref);
loaded_value.to_rvalue()
@ -980,7 +972,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
// TODO(antoyo): use ty.
// TODO(antoyo): handle alignment.
let atomic_load =
self.context.get_builtin_function(&format!("__atomic_load_{}", size.bytes()));
self.context.get_builtin_function(format!("__atomic_load_{}", size.bytes()));
let ordering = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc());
let volatile_const_void_ptr_type =
@ -1136,7 +1128,7 @@ impl<'a, 'gcc, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'gcc, 'tcx> {
) {
// TODO(antoyo): handle alignment.
let atomic_store =
self.context.get_builtin_function(&format!("__atomic_store_{}", size.bytes()));
self.context.get_builtin_function(format!("__atomic_store_{}", size.bytes()));
let ordering = self.context.new_rvalue_from_int(self.i32_type, order.to_gcc());
let volatile_const_void_ptr_type =
self.context.new_type::<()>().make_volatile().make_pointer();

8
compiler/rustc_codegen_gcc/src/consts.rs
View file

@ -9,6 +9,7 @@ use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}
use rustc_middle::mir::interpret::{
self, ConstAllocation, ErrorHandled, Scalar as InterpScalar, read_target_uint,
};
use rustc_middle::mir::mono::Linkage;
use rustc_middle::ty::layout::LayoutOf;
use rustc_middle::ty::{self, Instance};
use rustc_middle::{bug, span_bug};
@ -258,7 +259,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
if !self.tcx.is_reachable_non_generic(def_id) {
#[cfg(feature = "master")]
global.add_string_attribute(VarAttribute::Visibility(Visibility::Hidden));
global.add_attribute(VarAttribute::Visibility(Visibility::Hidden));
}
global
@ -386,6 +387,11 @@ fn check_and_apply_linkage<'gcc, 'tcx>(
let global1 =
cx.declare_global_with_linkage(sym, cx.type_i8(), base::global_linkage_to_gcc(linkage));
if linkage == Linkage::ExternalWeak {
#[cfg(feature = "master")]
global1.add_attribute(VarAttribute::Weak);
}
// Declare an internal global `extern_with_linkage_foo` which
// is initialized with the address of `foo`. If `foo` is
// discarded during linking (for example, if `foo` has weak

35
compiler/rustc_codegen_gcc/src/context.rs
View file

@ -24,6 +24,7 @@ use rustc_target::spec::{HasTargetSpec, HasWasmCAbiOpt, Target, TlsModel, WasmCA
use crate::callee::get_fn;
use crate::common::SignType;
#[cfg_attr(not(feature = "master"), allow(dead_code))]
pub struct CodegenCx<'gcc, 'tcx> {
pub codegen_unit: &'tcx CodegenUnit<'tcx>,
pub context: &'gcc Context<'gcc>,
@ -226,48 +227,14 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
"__builtin_umul_overflow",
"__builtin_usubll_overflow",
"__builtin_usub_overflow",
"sqrtf",
"sqrt",
"__builtin_powif",
"__builtin_powi",
"sinf",
"sin",
"cosf",
"cos",
"powf",
"pow",
"expf",
"exp",
"exp2f",
"exp2",
"logf",
"log",
"log10f",
"log10",
"log2f",
"log2",
"fmaf",
"fma",
"fabsf",
"fabs",
"fminf",
"fmin",
"fmaxf",
"fmax",
"copysignf",
"copysign",
"floorf",
"floor",
"ceilf",
"ceil",
"truncf",
"trunc",
"rintf",
"rint",
"nearbyintf",
"nearbyint",
"roundf",
"round",
];
for builtin in builtins.iter() {

4
compiler/rustc_codegen_gcc/src/debuginfo.rs
View file

@ -55,7 +55,7 @@ impl<'a, 'gcc, 'tcx> DebugInfoBuilderMethods for Builder<'a, 'gcc, 'tcx> {
}
/// Generate the `debug_context` in an MIR Body.
/// # Souce of Origin
/// # Source of Origin
/// Copied from `create_scope_map.rs` of rustc_codegen_llvm
fn compute_mir_scopes<'gcc, 'tcx>(
cx: &CodegenCx<'gcc, 'tcx>,
@ -90,7 +90,7 @@ fn compute_mir_scopes<'gcc, 'tcx>(
/// Update the `debug_context`, adding new scope to it,
/// if it's not added as is denoted in `instantiated`.
///
/// # Souce of Origin
/// # Source of Origin
/// Copied from `create_scope_map.rs` of rustc_codegen_llvm
/// FIXME(tempdragon/?): Add Scope Support Here.
fn make_mir_scope<'gcc, 'tcx>(

10
compiler/rustc_codegen_gcc/src/declare.rs
View file

@ -168,7 +168,15 @@ fn declare_raw_fn<'gcc>(
variadic: bool,
) -> Function<'gcc> {
if name.starts_with("llvm.") {
let intrinsic = llvm::intrinsic(name, cx);
let intrinsic = match name {
"llvm.fma.f16" => {
// fma is not a target builtin, but a normal builtin, so we handle it differently
// here.
cx.context.get_builtin_function("fma")
}
_ => llvm::intrinsic(name, cx),
};
cx.intrinsics.borrow_mut().insert(name.to_string(), intrinsic);
return intrinsic;
}

2
compiler/rustc_codegen_gcc/src/int.rs
View file

@ -733,7 +733,7 @@ impl<'a, 'gcc, 'tcx> Builder<'a, 'gcc, 'tcx> {
// TODO(antoyo): check if it's faster to use string literals and a
// match instead of format!.
let bswap = self.cx.context.get_builtin_function(&format!("__builtin_bswap{}", width));
let bswap = self.cx.context.get_builtin_function(format!("__builtin_bswap{}", width));
// FIXME(antoyo): this cast should not be necessary. Remove
// when having proper sized integer types.
let param_type = bswap.get_param(0).to_rvalue().get_type();

364
compiler/rustc_codegen_gcc/src/intrinsic/archs.rs
View file

@ -31,8 +31,11 @@ match name {
"llvm.AMDGPU.trig.preop.v2f64" => "__builtin_amdgpu_trig_preop",
"llvm.AMDGPU.trig.preop.v4f32" => "__builtin_amdgpu_trig_preop",
// aarch64
"llvm.aarch64.chkfeat" => "__builtin_arm_chkfeat",
"llvm.aarch64.dmb" => "__builtin_arm_dmb",
"llvm.aarch64.dsb" => "__builtin_arm_dsb",
"llvm.aarch64.gcspopm" => "__builtin_arm_gcspopm",
"llvm.aarch64.gcsss" => "__builtin_arm_gcsss",
"llvm.aarch64.isb" => "__builtin_arm_isb",
"llvm.aarch64.prefetch" => "__builtin_arm_prefetch",
"llvm.aarch64.sve.aesd" => "__builtin_sve_svaesd_u8",
@ -80,7 +83,6 @@ match name {
"llvm.amdgcn.dot4.f32.fp8.fp8" => "__builtin_amdgcn_dot4_f32_fp8_fp8",
"llvm.amdgcn.ds.add.gs.reg.rtn" => "__builtin_amdgcn_ds_add_gs_reg_rtn",
"llvm.amdgcn.ds.bpermute" => "__builtin_amdgcn_ds_bpermute",
"llvm.amdgcn.ds.fadd.v2bf16" => "__builtin_amdgcn_ds_atomic_fadd_v2bf16",
"llvm.amdgcn.ds.gws.barrier" => "__builtin_amdgcn_ds_gws_barrier",
"llvm.amdgcn.ds.gws.init" => "__builtin_amdgcn_ds_gws_init",
"llvm.amdgcn.ds.gws.sema.br" => "__builtin_amdgcn_ds_gws_sema_br",
@ -96,6 +98,7 @@ match name {
"llvm.amdgcn.fdot2.f16.f16" => "__builtin_amdgcn_fdot2_f16_f16",
"llvm.amdgcn.fdot2.f32.bf16" => "__builtin_amdgcn_fdot2_f32_bf16",
"llvm.amdgcn.fmul.legacy" => "__builtin_amdgcn_fmul_legacy",
"llvm.amdgcn.global.load.lds" => "__builtin_amdgcn_global_load_lds",
"llvm.amdgcn.groupstaticsize" => "__builtin_amdgcn_groupstaticsize",
"llvm.amdgcn.iglp.opt" => "__builtin_amdgcn_iglp_opt",
"llvm.amdgcn.implicit.buffer.ptr" => "__builtin_amdgcn_implicit_buffer_ptr",
@ -154,16 +157,11 @@ match name {
"llvm.amdgcn.mqsad.u32.u8" => "__builtin_amdgcn_mqsad_u32_u8",
"llvm.amdgcn.msad.u8" => "__builtin_amdgcn_msad_u8",
"llvm.amdgcn.perm" => "__builtin_amdgcn_perm",
"llvm.amdgcn.permlane16" => "__builtin_amdgcn_permlane16",
"llvm.amdgcn.permlane16.var" => "__builtin_amdgcn_permlane16_var",
"llvm.amdgcn.permlane64" => "__builtin_amdgcn_permlane64",
"llvm.amdgcn.permlanex16" => "__builtin_amdgcn_permlanex16",
"llvm.amdgcn.permlanex16.var" => "__builtin_amdgcn_permlanex16_var",
"llvm.amdgcn.qsad.pk.u16.u8" => "__builtin_amdgcn_qsad_pk_u16_u8",
"llvm.amdgcn.queue.ptr" => "__builtin_amdgcn_queue_ptr",
"llvm.amdgcn.rcp.legacy" => "__builtin_amdgcn_rcp_legacy",
"llvm.amdgcn.readfirstlane" => "__builtin_amdgcn_readfirstlane",
"llvm.amdgcn.readlane" => "__builtin_amdgcn_readlane",
"llvm.amdgcn.rsq.legacy" => "__builtin_amdgcn_rsq_legacy",
"llvm.amdgcn.s.barrier" => "__builtin_amdgcn_s_barrier",
"llvm.amdgcn.s.barrier.init" => "__builtin_amdgcn_s_barrier_init",
@ -192,6 +190,8 @@ match name {
"llvm.amdgcn.s.setreg" => "__builtin_amdgcn_s_setreg",
"llvm.amdgcn.s.sleep" => "__builtin_amdgcn_s_sleep",
"llvm.amdgcn.s.sleep.var" => "__builtin_amdgcn_s_sleep_var",
"llvm.amdgcn.s.ttracedata" => "__builtin_amdgcn_s_ttracedata",
"llvm.amdgcn.s.ttracedata.imm" => "__builtin_amdgcn_s_ttracedata_imm",
"llvm.amdgcn.s.wait.event.export.ready" => "__builtin_amdgcn_s_wait_event_export_ready",
"llvm.amdgcn.s.waitcnt" => "__builtin_amdgcn_s_waitcnt",
"llvm.amdgcn.s.wakeup.barrier" => "__builtin_amdgcn_s_wakeup_barrier",
@ -227,7 +227,6 @@ match name {
"llvm.amdgcn.workgroup.id.x" => "__builtin_amdgcn_workgroup_id_x",
"llvm.amdgcn.workgroup.id.y" => "__builtin_amdgcn_workgroup_id_y",
"llvm.amdgcn.workgroup.id.z" => "__builtin_amdgcn_workgroup_id_z",
"llvm.amdgcn.writelane" => "__builtin_amdgcn_writelane",
// arm
"llvm.arm.cdp" => "__builtin_arm_cdp",
"llvm.arm.cdp2" => "__builtin_arm_cdp2",
@ -4536,10 +4535,18 @@ match name {
"llvm.nvvm.div.rz.d" => "__nvvm_div_rz_d",
"llvm.nvvm.div.rz.f" => "__nvvm_div_rz_f",
"llvm.nvvm.div.rz.ftz.f" => "__nvvm_div_rz_ftz_f",
"llvm.nvvm.e4m3x2.to.f16x2.rn" => "__nvvm_e4m3x2_to_f16x2_rn",
"llvm.nvvm.e4m3x2.to.f16x2.rn.relu" => "__nvvm_e4m3x2_to_f16x2_rn_relu",
"llvm.nvvm.e5m2x2.to.f16x2.rn" => "__nvvm_e5m2x2_to_f16x2_rn",
"llvm.nvvm.e5m2x2.to.f16x2.rn.relu" => "__nvvm_e5m2x2_to_f16x2_rn_relu",
"llvm.nvvm.ex2.approx.d" => "__nvvm_ex2_approx_d",
"llvm.nvvm.ex2.approx.f" => "__nvvm_ex2_approx_f",
"llvm.nvvm.ex2.approx.ftz.f" => "__nvvm_ex2_approx_ftz_f",
"llvm.nvvm.exit" => "__nvvm_exit",
"llvm.nvvm.f16x2.to.e4m3x2.rn" => "__nvvm_f16x2_to_e4m3x2_rn",
"llvm.nvvm.f16x2.to.e4m3x2.rn.relu" => "__nvvm_f16x2_to_e4m3x2_rn_relu",
"llvm.nvvm.f16x2.to.e5m2x2.rn" => "__nvvm_f16x2_to_e5m2x2_rn",
"llvm.nvvm.f16x2.to.e5m2x2.rn.relu" => "__nvvm_f16x2_to_e5m2x2_rn_relu",
"llvm.nvvm.f2bf16.rn" => "__nvvm_f2bf16_rn",
"llvm.nvvm.f2bf16.rn.relu" => "__nvvm_f2bf16_rn_relu",
"llvm.nvvm.f2bf16.rz" => "__nvvm_f2bf16_rz",
@ -4582,6 +4589,10 @@ match name {
"llvm.nvvm.fabs.d" => "__nvvm_fabs_d",
"llvm.nvvm.fabs.f" => "__nvvm_fabs_f",
"llvm.nvvm.fabs.ftz.f" => "__nvvm_fabs_ftz_f",
"llvm.nvvm.ff.to.e4m3x2.rn" => "__nvvm_ff_to_e4m3x2_rn",
"llvm.nvvm.ff.to.e4m3x2.rn.relu" => "__nvvm_ff_to_e4m3x2_rn_relu",
"llvm.nvvm.ff.to.e5m2x2.rn" => "__nvvm_ff_to_e5m2x2_rn",
"llvm.nvvm.ff.to.e5m2x2.rn.relu" => "__nvvm_ff_to_e5m2x2_rn_relu",
"llvm.nvvm.ff2bf16x2.rn" => "__nvvm_ff2bf16x2_rn",
"llvm.nvvm.ff2bf16x2.rn.relu" => "__nvvm_ff2bf16x2_rn_relu",
"llvm.nvvm.ff2bf16x2.rz" => "__nvvm_ff2bf16x2_rz",
@ -4866,6 +4877,7 @@ match name {
"llvm.nvvm.round.ftz.f" => "__nvvm_round_ftz_f",
"llvm.nvvm.rsqrt.approx.d" => "__nvvm_rsqrt_approx_d",
"llvm.nvvm.rsqrt.approx.f" => "__nvvm_rsqrt_approx_f",
"llvm.nvvm.rsqrt.approx.ftz.d" => "__nvvm_rsqrt_approx_ftz_d",
"llvm.nvvm.rsqrt.approx.ftz.f" => "__nvvm_rsqrt_approx_ftz_f",
"llvm.nvvm.sad.i" => "__nvvm_sad_i",
"llvm.nvvm.sad.ll" => "__nvvm_sad_ll",
@ -5164,6 +5176,8 @@ match name {
// ppc
"llvm.ppc.addex" => "__builtin_ppc_addex",
"llvm.ppc.addf128.round.to.odd" => "__builtin_addf128_round_to_odd",
"llvm.ppc.addg6s" => "__builtin_addg6s",
"llvm.ppc.addg6sd" => "__builtin_ppc_addg6s",
"llvm.ppc.altivec.crypto.vcipher" => "__builtin_altivec_crypto_vcipher",
"llvm.ppc.altivec.crypto.vcipherlast" => "__builtin_altivec_crypto_vcipherlast",
"llvm.ppc.altivec.crypto.vncipher" => "__builtin_altivec_crypto_vncipher",
@ -5461,6 +5475,10 @@ match name {
"llvm.ppc.bcdsub" => "__builtin_ppc_bcdsub",
"llvm.ppc.bcdsub.p" => "__builtin_ppc_bcdsub_p",
"llvm.ppc.bpermd" => "__builtin_bpermd",
"llvm.ppc.cbcdtd" => "__builtin_cbcdtd",
"llvm.ppc.cbcdtdd" => "__builtin_ppc_cbcdtd",
"llvm.ppc.cdtbcd" => "__builtin_cdtbcd",
"llvm.ppc.cdtbcdd" => "__builtin_ppc_cdtbcd",
"llvm.ppc.cfuged" => "__builtin_cfuged",
"llvm.ppc.cmpeqb" => "__builtin_ppc_cmpeqb",
"llvm.ppc.cmprb" => "__builtin_ppc_cmprb",
@ -5627,7 +5645,6 @@ match name {
"llvm.ppc.qpx.qvstfs" => "__builtin_qpx_qvstfs",
"llvm.ppc.qpx.qvstfsa" => "__builtin_qpx_qvstfsa",
"llvm.ppc.readflm" => "__builtin_readflm",
"llvm.ppc.rldimi" => "__builtin_ppc_rldimi",
"llvm.ppc.rlwimi" => "__builtin_ppc_rlwimi",
"llvm.ppc.rlwnm" => "__builtin_ppc_rlwnm",
"llvm.ppc.scalar.extract.expq" => "__builtin_vsx_scalar_extract_expq",
@ -7210,29 +7227,6 @@ match name {
"llvm.ve.vl.xorm.MMM" => "__builtin_ve_vl_xorm_MMM",
"llvm.ve.vl.xorm.mmm" => "__builtin_ve_vl_xorm_mmm",
// x86
"llvm.x86.3dnow.pavgusb" => "__builtin_ia32_pavgusb",
"llvm.x86.3dnow.pf2id" => "__builtin_ia32_pf2id",
"llvm.x86.3dnow.pfacc" => "__builtin_ia32_pfacc",
"llvm.x86.3dnow.pfadd" => "__builtin_ia32_pfadd",
"llvm.x86.3dnow.pfcmpeq" => "__builtin_ia32_pfcmpeq",
"llvm.x86.3dnow.pfcmpge" => "__builtin_ia32_pfcmpge",
"llvm.x86.3dnow.pfcmpgt" => "__builtin_ia32_pfcmpgt",
"llvm.x86.3dnow.pfmax" => "__builtin_ia32_pfmax",
"llvm.x86.3dnow.pfmin" => "__builtin_ia32_pfmin",
"llvm.x86.3dnow.pfmul" => "__builtin_ia32_pfmul",
"llvm.x86.3dnow.pfrcp" => "__builtin_ia32_pfrcp",
"llvm.x86.3dnow.pfrcpit1" => "__builtin_ia32_pfrcpit1",
"llvm.x86.3dnow.pfrcpit2" => "__builtin_ia32_pfrcpit2",
"llvm.x86.3dnow.pfrsqit1" => "__builtin_ia32_pfrsqit1",
"llvm.x86.3dnow.pfrsqrt" => "__builtin_ia32_pfrsqrt",
"llvm.x86.3dnow.pfsub" => "__builtin_ia32_pfsub",
"llvm.x86.3dnow.pfsubr" => "__builtin_ia32_pfsubr",
"llvm.x86.3dnow.pi2fd" => "__builtin_ia32_pi2fd",
"llvm.x86.3dnow.pmulhrw" => "__builtin_ia32_pmulhrw",
"llvm.x86.3dnowa.pf2iw" => "__builtin_ia32_pf2iw",
"llvm.x86.3dnowa.pfnacc" => "__builtin_ia32_pfnacc",
"llvm.x86.3dnowa.pfpnacc" => "__builtin_ia32_pfpnacc",
"llvm.x86.3dnowa.pi2fw" => "__builtin_ia32_pi2fw",
"llvm.x86.aadd32" => "__builtin_ia32_aadd32",
"llvm.x86.aadd64" => "__builtin_ia32_aadd64",
"llvm.x86.aand32" => "__builtin_ia32_aand32",
@ -7334,6 +7328,207 @@ match name {
"llvm.x86.avx.vtestz.ps.256" => "__builtin_ia32_vtestzps256",
"llvm.x86.avx.vzeroall" => "__builtin_ia32_vzeroall",
"llvm.x86.avx.vzeroupper" => "__builtin_ia32_vzeroupper",
"llvm.x86.avx10.mask.vcvt2ps2phx.128" => "__builtin_ia32_vcvt2ps2phx128_mask",
"llvm.x86.avx10.mask.vcvt2ps2phx.256" => "__builtin_ia32_vcvt2ps2phx256_mask",
"llvm.x86.avx10.mask.vcvt2ps2phx.512" => "__builtin_ia32_vcvt2ps2phx512_mask",
"llvm.x86.avx10.mask.vcvtbiasph2bf8128" => "__builtin_ia32_vcvtbiasph2bf8_128_mask",
"llvm.x86.avx10.mask.vcvtbiasph2bf8256" => "__builtin_ia32_vcvtbiasph2bf8_256_mask",
"llvm.x86.avx10.mask.vcvtbiasph2bf8512" => "__builtin_ia32_vcvtbiasph2bf8_512_mask",
"llvm.x86.avx10.mask.vcvtbiasph2bf8s128" => "__builtin_ia32_vcvtbiasph2bf8s_128_mask",
"llvm.x86.avx10.mask.vcvtbiasph2bf8s256" => "__builtin_ia32_vcvtbiasph2bf8s_256_mask",
"llvm.x86.avx10.mask.vcvtbiasph2bf8s512" => "__builtin_ia32_vcvtbiasph2bf8s_512_mask",
"llvm.x86.avx10.mask.vcvtbiasph2hf8128" => "__builtin_ia32_vcvtbiasph2hf8_128_mask",
"llvm.x86.avx10.mask.vcvtbiasph2hf8256" => "__builtin_ia32_vcvtbiasph2hf8_256_mask",
"llvm.x86.avx10.mask.vcvtbiasph2hf8512" => "__builtin_ia32_vcvtbiasph2hf8_512_mask",
"llvm.x86.avx10.mask.vcvtbiasph2hf8s128" => "__builtin_ia32_vcvtbiasph2hf8s_128_mask",
"llvm.x86.avx10.mask.vcvtbiasph2hf8s256" => "__builtin_ia32_vcvtbiasph2hf8s_256_mask",
"llvm.x86.avx10.mask.vcvtbiasph2hf8s512" => "__builtin_ia32_vcvtbiasph2hf8s_512_mask",
"llvm.x86.avx10.mask.vcvthf82ph128" => "__builtin_ia32_vcvthf8_2ph128_mask",
"llvm.x86.avx10.mask.vcvthf82ph256" => "__builtin_ia32_vcvthf8_2ph256_mask",
"llvm.x86.avx10.mask.vcvthf82ph512" => "__builtin_ia32_vcvthf8_2ph512_mask",
"llvm.x86.avx10.mask.vcvtneph2bf8128" => "__builtin_ia32_vcvtneph2bf8_128_mask",
"llvm.x86.avx10.mask.vcvtneph2bf8256" => "__builtin_ia32_vcvtneph2bf8_256_mask",
"llvm.x86.avx10.mask.vcvtneph2bf8512" => "__builtin_ia32_vcvtneph2bf8_512_mask",
"llvm.x86.avx10.mask.vcvtneph2bf8s128" => "__builtin_ia32_vcvtneph2bf8s_128_mask",
"llvm.x86.avx10.mask.vcvtneph2bf8s256" => "__builtin_ia32_vcvtneph2bf8s_256_mask",
"llvm.x86.avx10.mask.vcvtneph2bf8s512" => "__builtin_ia32_vcvtneph2bf8s_512_mask",
"llvm.x86.avx10.mask.vcvtneph2hf8128" => "__builtin_ia32_vcvtneph2hf8_128_mask",
"llvm.x86.avx10.mask.vcvtneph2hf8256" => "__builtin_ia32_vcvtneph2hf8_256_mask",
"llvm.x86.avx10.mask.vcvtneph2hf8512" => "__builtin_ia32_vcvtneph2hf8_512_mask",
"llvm.x86.avx10.mask.vcvtneph2hf8s128" => "__builtin_ia32_vcvtneph2hf8s_128_mask",
"llvm.x86.avx10.mask.vcvtneph2hf8s256" => "__builtin_ia32_vcvtneph2hf8s_256_mask",
"llvm.x86.avx10.mask.vcvtneph2hf8s512" => "__builtin_ia32_vcvtneph2hf8s_512_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2dq256" => "__builtin_ia32_vcvtpd2dq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2ph256" => "__builtin_ia32_vcvtpd2ph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2ps256" => "__builtin_ia32_vcvtpd2ps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2qq256" => "__builtin_ia32_vcvtpd2qq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2udq256" => "__builtin_ia32_vcvtpd2udq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtpd2uqq256" => "__builtin_ia32_vcvtpd2uqq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2dq256" => "__builtin_ia32_vcvtph2dq256_round_mask",
"llvm.x86.avx10.mask.vcvtph2ibs128" => "__builtin_ia32_vcvtph2ibs128_mask",
"llvm.x86.avx10.mask.vcvtph2ibs256" => "__builtin_ia32_vcvtph2ibs256_mask",
"llvm.x86.avx10.mask.vcvtph2ibs512" => "__builtin_ia32_vcvtph2ibs512_mask",
"llvm.x86.avx10.mask.vcvtph2iubs128" => "__builtin_ia32_vcvtph2iubs128_mask",
"llvm.x86.avx10.mask.vcvtph2iubs256" => "__builtin_ia32_vcvtph2iubs256_mask",
"llvm.x86.avx10.mask.vcvtph2iubs512" => "__builtin_ia32_vcvtph2iubs512_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2pd256" => "__builtin_ia32_vcvtph2pd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2psx256" => "__builtin_ia32_vcvtph2psx256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2qq256" => "__builtin_ia32_vcvtph2qq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2udq256" => "__builtin_ia32_vcvtph2udq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2uqq256" => "__builtin_ia32_vcvtph2uqq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2uw256" => "__builtin_ia32_vcvtph2uw256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtph2w256" => "__builtin_ia32_vcvtph2w256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2dq256" => "__builtin_ia32_vcvtps2dq256_round_mask",
"llvm.x86.avx10.mask.vcvtps2ibs128" => "__builtin_ia32_vcvtps2ibs128_mask",
"llvm.x86.avx10.mask.vcvtps2ibs256" => "__builtin_ia32_vcvtps2ibs256_mask",
"llvm.x86.avx10.mask.vcvtps2ibs512" => "__builtin_ia32_vcvtps2ibs512_mask",
"llvm.x86.avx10.mask.vcvtps2iubs128" => "__builtin_ia32_vcvtps2iubs128_mask",
"llvm.x86.avx10.mask.vcvtps2iubs256" => "__builtin_ia32_vcvtps2iubs256_mask",
"llvm.x86.avx10.mask.vcvtps2iubs512" => "__builtin_ia32_vcvtps2iubs512_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2pd256" => "__builtin_ia32_vcvtps2pd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2ph256" => "__builtin_ia32_vcvtps2ph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2phx256" => "__builtin_ia32_vcvtps2phx256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2qq256" => "__builtin_ia32_vcvtps2qq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2udq256" => "__builtin_ia32_vcvtps2udq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvtps2uqq256" => "__builtin_ia32_vcvtps2uqq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttpd2dq256" => "__builtin_ia32_vcvttpd2dq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttpd2qq256" => "__builtin_ia32_vcvttpd2qq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttpd2udq256" => "__builtin_ia32_vcvttpd2udq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttpd2uqq256" => "__builtin_ia32_vcvttpd2uqq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2dq256" => "__builtin_ia32_vcvttph2dq256_round_mask",
"llvm.x86.avx10.mask.vcvttph2ibs128" => "__builtin_ia32_vcvttph2ibs128_mask",
"llvm.x86.avx10.mask.vcvttph2ibs256" => "__builtin_ia32_vcvttph2ibs256_mask",
"llvm.x86.avx10.mask.vcvttph2ibs512" => "__builtin_ia32_vcvttph2ibs512_mask",
"llvm.x86.avx10.mask.vcvttph2iubs128" => "__builtin_ia32_vcvttph2iubs128_mask",
"llvm.x86.avx10.mask.vcvttph2iubs256" => "__builtin_ia32_vcvttph2iubs256_mask",
"llvm.x86.avx10.mask.vcvttph2iubs512" => "__builtin_ia32_vcvttph2iubs512_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2qq256" => "__builtin_ia32_vcvttph2qq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2udq256" => "__builtin_ia32_vcvttph2udq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2uqq256" => "__builtin_ia32_vcvttph2uqq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2uw256" => "__builtin_ia32_vcvttph2uw256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttph2w256" => "__builtin_ia32_vcvttph2w256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttps2dq256" => "__builtin_ia32_vcvttps2dq256_round_mask",
"llvm.x86.avx10.mask.vcvttps2ibs128" => "__builtin_ia32_vcvttps2ibs128_mask",
"llvm.x86.avx10.mask.vcvttps2ibs256" => "__builtin_ia32_vcvttps2ibs256_mask",
"llvm.x86.avx10.mask.vcvttps2ibs512" => "__builtin_ia32_vcvttps2ibs512_mask",
"llvm.x86.avx10.mask.vcvttps2iubs128" => "__builtin_ia32_vcvttps2iubs128_mask",
"llvm.x86.avx10.mask.vcvttps2iubs256" => "__builtin_ia32_vcvttps2iubs256_mask",
"llvm.x86.avx10.mask.vcvttps2iubs512" => "__builtin_ia32_vcvttps2iubs512_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttps2qq256" => "__builtin_ia32_vcvttps2qq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttps2udq256" => "__builtin_ia32_vcvttps2udq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vcvttps2uqq256" => "__builtin_ia32_vcvttps2uqq256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfcmaddcph256" => "__builtin_ia32_vfcmaddcph256_round_mask3",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfcmulcph256" => "__builtin_ia32_vfcmulcph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfixupimmpd256" => "__builtin_ia32_vfixupimmpd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfixupimmps256" => "__builtin_ia32_vfixupimmps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfmaddcph256" => "__builtin_ia32_vfmaddcph256_round_mask3",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vfmulcph256" => "__builtin_ia32_vfmulcph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetexppd256" => "__builtin_ia32_vgetexppd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetexpph256" => "__builtin_ia32_vgetexpph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetexpps256" => "__builtin_ia32_vgetexpps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetmantpd256" => "__builtin_ia32_vgetmantpd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetmantph256" => "__builtin_ia32_vgetmantph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vgetmantps256" => "__builtin_ia32_vgetmantps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxpd.round" => "__builtin_ia32_vminmaxpd512_round_mask",
"llvm.x86.avx10.mask.vminmaxpd128" => "__builtin_ia32_vminmaxpd128_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxpd256.round" => "__builtin_ia32_vminmaxpd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxph.round" => "__builtin_ia32_vminmaxph512_round_mask",
"llvm.x86.avx10.mask.vminmaxph128" => "__builtin_ia32_vminmaxph128_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxph256.round" => "__builtin_ia32_vminmaxph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxps.round" => "__builtin_ia32_vminmaxps512_round_mask",
"llvm.x86.avx10.mask.vminmaxps128" => "__builtin_ia32_vminmaxps128_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxps256.round" => "__builtin_ia32_vminmaxps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxsd.round" => "__builtin_ia32_vminmaxsd_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxsh.round" => "__builtin_ia32_vminmaxsh_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vminmaxss.round" => "__builtin_ia32_vminmaxss_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrangepd256" => "__builtin_ia32_vrangepd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrangeps256" => "__builtin_ia32_vrangeps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vreducepd256" => "__builtin_ia32_vreducepd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vreduceph256" => "__builtin_ia32_vreduceph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vreduceps256" => "__builtin_ia32_vreduceps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrndscalepd256" => "__builtin_ia32_vrndscalepd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrndscaleph256" => "__builtin_ia32_vrndscaleph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vrndscaleps256" => "__builtin_ia32_vrndscaleps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vscalefpd256" => "__builtin_ia32_vscalefpd256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vscalefph256" => "__builtin_ia32_vscalefph256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.mask.vscalefps256" => "__builtin_ia32_vscalefps256_round_mask",
// [INVALID CONVERSION]: "llvm.x86.avx10.maskz.vfcmaddcph256" => "__builtin_ia32_vfcmaddcph256_round_maskz",
// [INVALID CONVERSION]: "llvm.x86.avx10.maskz.vfixupimmpd256" => "__builtin_ia32_vfixupimmpd256_round_maskz",
// [INVALID CONVERSION]: "llvm.x86.avx10.maskz.vfixupimmps256" => "__builtin_ia32_vfixupimmps256_round_maskz",
// [INVALID CONVERSION]: "llvm.x86.avx10.maskz.vfmaddcph256" => "__builtin_ia32_vfmaddcph256_round_maskz",
"llvm.x86.avx10.vaddpd256" => "__builtin_ia32_vaddpd256_round",
"llvm.x86.avx10.vaddph256" => "__builtin_ia32_vaddph256_round",
"llvm.x86.avx10.vaddps256" => "__builtin_ia32_vaddps256_round",
"llvm.x86.avx10.vcvtne2ph2bf8128" => "__builtin_ia32_vcvtne2ph2bf8_128",
"llvm.x86.avx10.vcvtne2ph2bf8256" => "__builtin_ia32_vcvtne2ph2bf8_256",
"llvm.x86.avx10.vcvtne2ph2bf8512" => "__builtin_ia32_vcvtne2ph2bf8_512",
"llvm.x86.avx10.vcvtne2ph2bf8s128" => "__builtin_ia32_vcvtne2ph2bf8s_128",
"llvm.x86.avx10.vcvtne2ph2bf8s256" => "__builtin_ia32_vcvtne2ph2bf8s_256",
"llvm.x86.avx10.vcvtne2ph2bf8s512" => "__builtin_ia32_vcvtne2ph2bf8s_512",
"llvm.x86.avx10.vcvtne2ph2hf8128" => "__builtin_ia32_vcvtne2ph2hf8_128",
"llvm.x86.avx10.vcvtne2ph2hf8256" => "__builtin_ia32_vcvtne2ph2hf8_256",
"llvm.x86.avx10.vcvtne2ph2hf8512" => "__builtin_ia32_vcvtne2ph2hf8_512",
"llvm.x86.avx10.vcvtne2ph2hf8s128" => "__builtin_ia32_vcvtne2ph2hf8s_128",
"llvm.x86.avx10.vcvtne2ph2hf8s256" => "__builtin_ia32_vcvtne2ph2hf8s_256",
"llvm.x86.avx10.vcvtne2ph2hf8s512" => "__builtin_ia32_vcvtne2ph2hf8s_512",
"llvm.x86.avx10.vcvtnebf162ibs128" => "__builtin_ia32_vcvtnebf162ibs128",
"llvm.x86.avx10.vcvtnebf162ibs256" => "__builtin_ia32_vcvtnebf162ibs256",
"llvm.x86.avx10.vcvtnebf162ibs512" => "__builtin_ia32_vcvtnebf162ibs512",
"llvm.x86.avx10.vcvtnebf162iubs128" => "__builtin_ia32_vcvtnebf162iubs128",
"llvm.x86.avx10.vcvtnebf162iubs256" => "__builtin_ia32_vcvtnebf162iubs256",
"llvm.x86.avx10.vcvtnebf162iubs512" => "__builtin_ia32_vcvtnebf162iubs512",
"llvm.x86.avx10.vcvttnebf162ibs128" => "__builtin_ia32_vcvttnebf162ibs128",
"llvm.x86.avx10.vcvttnebf162ibs256" => "__builtin_ia32_vcvttnebf162ibs256",
"llvm.x86.avx10.vcvttnebf162ibs512" => "__builtin_ia32_vcvttnebf162ibs512",
"llvm.x86.avx10.vcvttnebf162iubs128" => "__builtin_ia32_vcvttnebf162iubs128",
"llvm.x86.avx10.vcvttnebf162iubs256" => "__builtin_ia32_vcvttnebf162iubs256",
"llvm.x86.avx10.vcvttnebf162iubs512" => "__builtin_ia32_vcvttnebf162iubs512",
"llvm.x86.avx10.vdivpd256" => "__builtin_ia32_vdivpd256_round",
"llvm.x86.avx10.vdivph256" => "__builtin_ia32_vdivph256_round",
"llvm.x86.avx10.vdivps256" => "__builtin_ia32_vdivps256_round",
"llvm.x86.avx10.vdpphps.128" => "__builtin_ia32_vdpphps128",
"llvm.x86.avx10.vdpphps.256" => "__builtin_ia32_vdpphps256",
"llvm.x86.avx10.vdpphps.512" => "__builtin_ia32_vdpphps512",
"llvm.x86.avx10.vfmaddsubpd256" => "__builtin_ia32_vfmaddsubpd256_round",
"llvm.x86.avx10.vfmaddsubph256" => "__builtin_ia32_vfmaddsubph256_round",
"llvm.x86.avx10.vfmaddsubps256" => "__builtin_ia32_vfmaddsubps256_round",
"llvm.x86.avx10.vmaxpd256" => "__builtin_ia32_vmaxpd256_round",
"llvm.x86.avx10.vmaxph256" => "__builtin_ia32_vmaxph256_round",
"llvm.x86.avx10.vmaxps256" => "__builtin_ia32_vmaxps256_round",
"llvm.x86.avx10.vminmaxnepbf16128" => "__builtin_ia32_vminmaxnepbf16128",
"llvm.x86.avx10.vminmaxnepbf16256" => "__builtin_ia32_vminmaxnepbf16256",
"llvm.x86.avx10.vminmaxnepbf16512" => "__builtin_ia32_vminmaxnepbf16512",
"llvm.x86.avx10.vminmaxpd128" => "__builtin_ia32_vminmaxpd128",
"llvm.x86.avx10.vminmaxpd256" => "__builtin_ia32_vminmaxpd256",
"llvm.x86.avx10.vminmaxph128" => "__builtin_ia32_vminmaxph128",
"llvm.x86.avx10.vminmaxph256" => "__builtin_ia32_vminmaxph256",
"llvm.x86.avx10.vminmaxps128" => "__builtin_ia32_vminmaxps128",
"llvm.x86.avx10.vminmaxps256" => "__builtin_ia32_vminmaxps256",
"llvm.x86.avx10.vminpd256" => "__builtin_ia32_vminpd256_round",
"llvm.x86.avx10.vminph256" => "__builtin_ia32_vminph256_round",
"llvm.x86.avx10.vminps256" => "__builtin_ia32_vminps256_round",
"llvm.x86.avx10.vmpsadbw.512" => "__builtin_ia32_mpsadbw512",
"llvm.x86.avx10.vmulpd256" => "__builtin_ia32_vmulpd256_round",
"llvm.x86.avx10.vmulph256" => "__builtin_ia32_vmulph256_round",
"llvm.x86.avx10.vmulps256" => "__builtin_ia32_vmulps256_round",
"llvm.x86.avx10.vpdpbssd.512" => "__builtin_ia32_vpdpbssd512",
"llvm.x86.avx10.vpdpbssds.512" => "__builtin_ia32_vpdpbssds512",
"llvm.x86.avx10.vpdpbsud.512" => "__builtin_ia32_vpdpbsud512",
"llvm.x86.avx10.vpdpbsuds.512" => "__builtin_ia32_vpdpbsuds512",
"llvm.x86.avx10.vpdpbuud.512" => "__builtin_ia32_vpdpbuud512",
"llvm.x86.avx10.vpdpbuuds.512" => "__builtin_ia32_vpdpbuuds512",
"llvm.x86.avx10.vpdpwsud.512" => "__builtin_ia32_vpdpwsud512",
"llvm.x86.avx10.vpdpwsuds.512" => "__builtin_ia32_vpdpwsuds512",
"llvm.x86.avx10.vpdpwusd.512" => "__builtin_ia32_vpdpwusd512",
"llvm.x86.avx10.vpdpwusds.512" => "__builtin_ia32_vpdpwusds512",
"llvm.x86.avx10.vpdpwuud.512" => "__builtin_ia32_vpdpwuud512",
"llvm.x86.avx10.vpdpwuuds.512" => "__builtin_ia32_vpdpwuuds512",
"llvm.x86.avx10.vsqrtpd256" => "__builtin_ia32_vsqrtpd256_round",
"llvm.x86.avx10.vsqrtph256" => "__builtin_ia32_vsqrtph256_round",
"llvm.x86.avx10.vsqrtps256" => "__builtin_ia32_vsqrtps256_round",
"llvm.x86.avx10.vsubpd256" => "__builtin_ia32_vsubpd256_round",
"llvm.x86.avx10.vsubph256" => "__builtin_ia32_vsubph256_round",
"llvm.x86.avx10.vsubps256" => "__builtin_ia32_vsubps256_round",
"llvm.x86.avx2.gather.d.d" => "__builtin_ia32_gatherd_d",
"llvm.x86.avx2.gather.d.d.256" => "__builtin_ia32_gatherd_d256",
"llvm.x86.avx2.gather.d.pd" => "__builtin_ia32_gatherd_pd",
@ -8738,10 +8933,10 @@ match name {
"llvm.x86.avx512.rcp14.ss" => "__builtin_ia32_rcp14ss_mask",
"llvm.x86.avx512.rcp28.pd" => "__builtin_ia32_rcp28pd_mask",
"llvm.x86.avx512.rcp28.ps" => "__builtin_ia32_rcp28ps_mask",
// [INVALID CONVERSION]: "llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_round_mask",
// [DUPLICATE]: "llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_mask",
// [INVALID CONVERSION]: "llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_round_mask",
// [DUPLICATE]: "llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_mask",
"llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_mask",
// [DUPLICATE]: "llvm.x86.avx512.rcp28.sd" => "__builtin_ia32_rcp28sd_round_mask",
"llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_mask",
// [DUPLICATE]: "llvm.x86.avx512.rcp28.ss" => "__builtin_ia32_rcp28ss_round_mask",
"llvm.x86.avx512.rndscale.sd" => "__builtin_ia32_rndscalesd",
"llvm.x86.avx512.rndscale.ss" => "__builtin_ia32_rndscaless",
"llvm.x86.avx512.rsqrt14.pd.128" => "__builtin_ia32_rsqrt14pd128_mask",
@ -8754,10 +8949,10 @@ match name {
"llvm.x86.avx512.rsqrt14.ss" => "__builtin_ia32_rsqrt14ss_mask",
"llvm.x86.avx512.rsqrt28.pd" => "__builtin_ia32_rsqrt28pd_mask",
"llvm.x86.avx512.rsqrt28.ps" => "__builtin_ia32_rsqrt28ps_mask",
// [INVALID CONVERSION]: "llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_round_mask",
// [DUPLICATE]: "llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_mask",
// [INVALID CONVERSION]: "llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_round_mask",
// [DUPLICATE]: "llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_mask",
"llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_mask",
// [DUPLICATE]: "llvm.x86.avx512.rsqrt28.sd" => "__builtin_ia32_rsqrt28sd_round_mask",
"llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_mask",
// [DUPLICATE]: "llvm.x86.avx512.rsqrt28.ss" => "__builtin_ia32_rsqrt28ss_round_mask",
"llvm.x86.avx512.scatter.dpd.512" => "__builtin_ia32_scattersiv8df",
"llvm.x86.avx512.scatter.dpi.512" => "__builtin_ia32_scattersiv16si",
"llvm.x86.avx512.scatter.dpq.512" => "__builtin_ia32_scattersiv8di",
@ -9082,75 +9277,6 @@ match name {
"llvm.x86.lwpval64" => "__builtin_ia32_lwpval64",
"llvm.x86.mmx.emms" => "__builtin_ia32_emms",
"llvm.x86.mmx.femms" => "__builtin_ia32_femms",
"llvm.x86.mmx.maskmovq" => "__builtin_ia32_maskmovq",
"llvm.x86.mmx.movnt.dq" => "__builtin_ia32_movntq",
"llvm.x86.mmx.packssdw" => "__builtin_ia32_packssdw",
"llvm.x86.mmx.packsswb" => "__builtin_ia32_packsswb",
"llvm.x86.mmx.packuswb" => "__builtin_ia32_packuswb",
"llvm.x86.mmx.padd.b" => "__builtin_ia32_paddb",
"llvm.x86.mmx.padd.d" => "__builtin_ia32_paddd",
"llvm.x86.mmx.padd.q" => "__builtin_ia32_paddq",
"llvm.x86.mmx.padd.w" => "__builtin_ia32_paddw",
"llvm.x86.mmx.padds.b" => "__builtin_ia32_paddsb",
"llvm.x86.mmx.padds.w" => "__builtin_ia32_paddsw",
"llvm.x86.mmx.paddus.b" => "__builtin_ia32_paddusb",
"llvm.x86.mmx.paddus.w" => "__builtin_ia32_paddusw",
"llvm.x86.mmx.palignr.b" => "__builtin_ia32_palignr",
"llvm.x86.mmx.pand" => "__builtin_ia32_pand",
"llvm.x86.mmx.pandn" => "__builtin_ia32_pandn",
"llvm.x86.mmx.pavg.b" => "__builtin_ia32_pavgb",
"llvm.x86.mmx.pavg.w" => "__builtin_ia32_pavgw",
"llvm.x86.mmx.pcmpeq.b" => "__builtin_ia32_pcmpeqb",
"llvm.x86.mmx.pcmpeq.d" => "__builtin_ia32_pcmpeqd",
"llvm.x86.mmx.pcmpeq.w" => "__builtin_ia32_pcmpeqw",
"llvm.x86.mmx.pcmpgt.b" => "__builtin_ia32_pcmpgtb",
"llvm.x86.mmx.pcmpgt.d" => "__builtin_ia32_pcmpgtd",
"llvm.x86.mmx.pcmpgt.w" => "__builtin_ia32_pcmpgtw",
"llvm.x86.mmx.pextr.w" => "__builtin_ia32_vec_ext_v4hi",
"llvm.x86.mmx.pinsr.w" => "__builtin_ia32_vec_set_v4hi",
"llvm.x86.mmx.pmadd.wd" => "__builtin_ia32_pmaddwd",
"llvm.x86.mmx.pmaxs.w" => "__builtin_ia32_pmaxsw",
"llvm.x86.mmx.pmaxu.b" => "__builtin_ia32_pmaxub",
"llvm.x86.mmx.pmins.w" => "__builtin_ia32_pminsw",
"llvm.x86.mmx.pminu.b" => "__builtin_ia32_pminub",
"llvm.x86.mmx.pmovmskb" => "__builtin_ia32_pmovmskb",
"llvm.x86.mmx.pmulh.w" => "__builtin_ia32_pmulhw",
"llvm.x86.mmx.pmulhu.w" => "__builtin_ia32_pmulhuw",
"llvm.x86.mmx.pmull.w" => "__builtin_ia32_pmullw",
"llvm.x86.mmx.pmulu.dq" => "__builtin_ia32_pmuludq",
"llvm.x86.mmx.por" => "__builtin_ia32_por",
"llvm.x86.mmx.psad.bw" => "__builtin_ia32_psadbw",
"llvm.x86.mmx.psll.d" => "__builtin_ia32_pslld",
"llvm.x86.mmx.psll.q" => "__builtin_ia32_psllq",
"llvm.x86.mmx.psll.w" => "__builtin_ia32_psllw",
"llvm.x86.mmx.pslli.d" => "__builtin_ia32_pslldi",
"llvm.x86.mmx.pslli.q" => "__builtin_ia32_psllqi",
"llvm.x86.mmx.pslli.w" => "__builtin_ia32_psllwi",
"llvm.x86.mmx.psra.d" => "__builtin_ia32_psrad",
"llvm.x86.mmx.psra.w" => "__builtin_ia32_psraw",
"llvm.x86.mmx.psrai.d" => "__builtin_ia32_psradi",
"llvm.x86.mmx.psrai.w" => "__builtin_ia32_psrawi",
"llvm.x86.mmx.psrl.d" => "__builtin_ia32_psrld",
"llvm.x86.mmx.psrl.q" => "__builtin_ia32_psrlq",
"llvm.x86.mmx.psrl.w" => "__builtin_ia32_psrlw",
"llvm.x86.mmx.psrli.d" => "__builtin_ia32_psrldi",
"llvm.x86.mmx.psrli.q" => "__builtin_ia32_psrlqi",
"llvm.x86.mmx.psrli.w" => "__builtin_ia32_psrlwi",
"llvm.x86.mmx.psub.b" => "__builtin_ia32_psubb",
"llvm.x86.mmx.psub.d" => "__builtin_ia32_psubd",
"llvm.x86.mmx.psub.q" => "__builtin_ia32_psubq",
"llvm.x86.mmx.psub.w" => "__builtin_ia32_psubw",
"llvm.x86.mmx.psubs.b" => "__builtin_ia32_psubsb",
"llvm.x86.mmx.psubs.w" => "__builtin_ia32_psubsw",
"llvm.x86.mmx.psubus.b" => "__builtin_ia32_psubusb",
"llvm.x86.mmx.psubus.w" => "__builtin_ia32_psubusw",
"llvm.x86.mmx.punpckhbw" => "__builtin_ia32_punpckhbw",
"llvm.x86.mmx.punpckhdq" => "__builtin_ia32_punpckhdq",
"llvm.x86.mmx.punpckhwd" => "__builtin_ia32_punpckhwd",
"llvm.x86.mmx.punpcklbw" => "__builtin_ia32_punpcklbw",
"llvm.x86.mmx.punpckldq" => "__builtin_ia32_punpckldq",
"llvm.x86.mmx.punpcklwd" => "__builtin_ia32_punpcklwd",
"llvm.x86.mmx.pxor" => "__builtin_ia32_pxor",
"llvm.x86.monitorx" => "__builtin_ia32_monitorx",
"llvm.x86.movdir64b" => "__builtin_ia32_movdir64b",
"llvm.x86.mwaitx" => "__builtin_ia32_mwaitx",
@ -9193,16 +9319,10 @@ match name {
"llvm.x86.sse.comile.ss" => "__builtin_ia32_comile",
"llvm.x86.sse.comilt.ss" => "__builtin_ia32_comilt",
"llvm.x86.sse.comineq.ss" => "__builtin_ia32_comineq",
"llvm.x86.sse.cvtpd2pi" => "__builtin_ia32_cvtpd2pi",
"llvm.x86.sse.cvtpi2pd" => "__builtin_ia32_cvtpi2pd",
"llvm.x86.sse.cvtpi2ps" => "__builtin_ia32_cvtpi2ps",
"llvm.x86.sse.cvtps2pi" => "__builtin_ia32_cvtps2pi",
"llvm.x86.sse.cvtsi2ss" => "__builtin_ia32_cvtsi2ss",
"llvm.x86.sse.cvtsi642ss" => "__builtin_ia32_cvtsi642ss",
"llvm.x86.sse.cvtss2si" => "__builtin_ia32_cvtss2si",
"llvm.x86.sse.cvtss2si64" => "__builtin_ia32_cvtss2si64",
"llvm.x86.sse.cvttpd2pi" => "__builtin_ia32_cvttpd2pi",
"llvm.x86.sse.cvttps2pi" => "__builtin_ia32_cvttps2pi",
"llvm.x86.sse.cvttss2si" => "__builtin_ia32_cvttss2si",
"llvm.x86.sse.cvttss2si64" => "__builtin_ia32_cvttss2si64",
"llvm.x86.sse.div.ss" => "__builtin_ia32_divss",
@ -9212,7 +9332,6 @@ match name {
"llvm.x86.sse.min.ss" => "__builtin_ia32_minss",
"llvm.x86.sse.movmsk.ps" => "__builtin_ia32_movmskps",
"llvm.x86.sse.mul.ss" => "__builtin_ia32_mulss",
"llvm.x86.sse.pshuf.w" => "__builtin_ia32_pshufw",
"llvm.x86.sse.rcp.ps" => "__builtin_ia32_rcpps",
"llvm.x86.sse.rcp.ss" => "__builtin_ia32_rcpss",
"llvm.x86.sse.rsqrt.ps" => "__builtin_ia32_rsqrtps",
@ -9398,35 +9517,20 @@ match name {
"llvm.x86.sse4a.insertqi" => "__builtin_ia32_insertqi",
"llvm.x86.sse4a.movnt.sd" => "__builtin_ia32_movntsd",
"llvm.x86.sse4a.movnt.ss" => "__builtin_ia32_movntss",
"llvm.x86.ssse3.pabs.b" => "__builtin_ia32_pabsb",
"llvm.x86.ssse3.pabs.b.128" => "__builtin_ia32_pabsb128",
"llvm.x86.ssse3.pabs.d" => "__builtin_ia32_pabsd",
"llvm.x86.ssse3.pabs.d.128" => "__builtin_ia32_pabsd128",
"llvm.x86.ssse3.pabs.w" => "__builtin_ia32_pabsw",
"llvm.x86.ssse3.pabs.w.128" => "__builtin_ia32_pabsw128",
"llvm.x86.ssse3.phadd.d" => "__builtin_ia32_phaddd",
"llvm.x86.ssse3.phadd.d.128" => "__builtin_ia32_phaddd128",
"llvm.x86.ssse3.phadd.sw" => "__builtin_ia32_phaddsw",
"llvm.x86.ssse3.phadd.sw.128" => "__builtin_ia32_phaddsw128",
"llvm.x86.ssse3.phadd.w" => "__builtin_ia32_phaddw",
"llvm.x86.ssse3.phadd.w.128" => "__builtin_ia32_phaddw128",
"llvm.x86.ssse3.phsub.d" => "__builtin_ia32_phsubd",
"llvm.x86.ssse3.phsub.d.128" => "__builtin_ia32_phsubd128",
"llvm.x86.ssse3.phsub.sw" => "__builtin_ia32_phsubsw",
"llvm.x86.ssse3.phsub.sw.128" => "__builtin_ia32_phsubsw128",
"llvm.x86.ssse3.phsub.w" => "__builtin_ia32_phsubw",
"llvm.x86.ssse3.phsub.w.128" => "__builtin_ia32_phsubw128",
"llvm.x86.ssse3.pmadd.ub.sw" => "__builtin_ia32_pmaddubsw",
"llvm.x86.ssse3.pmadd.ub.sw.128" => "__builtin_ia32_pmaddubsw128",
"llvm.x86.ssse3.pmul.hr.sw" => "__builtin_ia32_pmulhrsw",
"llvm.x86.ssse3.pmul.hr.sw.128" => "__builtin_ia32_pmulhrsw128",
"llvm.x86.ssse3.pshuf.b" => "__builtin_ia32_pshufb",
"llvm.x86.ssse3.pshuf.b.128" => "__builtin_ia32_pshufb128",
"llvm.x86.ssse3.psign.b" => "__builtin_ia32_psignb",
"llvm.x86.ssse3.psign.b.128" => "__builtin_ia32_psignb128",
"llvm.x86.ssse3.psign.d" => "__builtin_ia32_psignd",
"llvm.x86.ssse3.psign.d.128" => "__builtin_ia32_psignd128",
"llvm.x86.ssse3.psign.w" => "__builtin_ia32_psignw",
"llvm.x86.ssse3.psign.w.128" => "__builtin_ia32_psignw128",
"llvm.x86.sttilecfg" => "__builtin_ia32_tile_storeconfig",
"llvm.x86.stui" => "__builtin_ia32_stui",

384
compiler/rustc_codegen_gcc/src/intrinsic/llvm.rs
View file

@ -1,11 +1,43 @@
use std::borrow::Cow;
use gccjit::{Function, FunctionPtrType, RValue, ToRValue, UnaryOp};
use gccjit::{CType, Context, Function, FunctionPtrType, RValue, ToRValue, UnaryOp};
use rustc_codegen_ssa::traits::BuilderMethods;
use crate::builder::Builder;
use crate::context::CodegenCx;
#[cfg_attr(not(feature = "master"), allow(unused_variables))]
pub fn adjust_function<'gcc>(
context: &'gcc Context<'gcc>,
func_name: &str,
func_ptr: RValue<'gcc>,
args: &[RValue<'gcc>],
) -> RValue<'gcc> {
// FIXME: we should not need this hack: this is required because both _mm_fcmadd_sch
// and _mm_mask3_fcmadd_round_sch calls llvm.x86.avx512fp16.mask.vfcmadd.csh and we
// seem to need to map this one LLVM intrinsic to 2 different GCC builtins.
#[cfg(feature = "master")]
match func_name {
"__builtin_ia32_vfcmaddcsh_mask3_round" => {
if format!("{:?}", args[3]).ends_with("255") {
return context
.get_target_builtin_function("__builtin_ia32_vfcmaddcsh_mask_round")
.get_address(None);
}
}
"__builtin_ia32_vfmaddcsh_mask3_round" => {
if format!("{:?}", args[3]).ends_with("255") {
return context
.get_target_builtin_function("__builtin_ia32_vfmaddcsh_mask_round")
.get_address(None);
}
}
_ => (),
}
func_ptr
}
pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
builder: &Builder<'a, 'gcc, 'tcx>,
gcc_func: FunctionPtrType<'gcc>,
@ -13,6 +45,11 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
func_name: &str,
original_function_name: Option<&String>,
) -> Cow<'b, [RValue<'gcc>]> {
// TODO: this might not be a good way to workaround the missing tile builtins.
if func_name == "__builtin_trap" {
return vec![].into();
}
// Some LLVM intrinsics do not map 1-to-1 to GCC intrinsics, so we add the missing
// arguments here.
if gcc_func.get_param_count() != args.len() {
@ -147,7 +184,11 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
| "__builtin_ia32_psrav16hi_mask"
| "__builtin_ia32_psrav8hi_mask"
| "__builtin_ia32_permvarhi256_mask"
| "__builtin_ia32_permvarhi128_mask" => {
| "__builtin_ia32_permvarhi128_mask"
| "__builtin_ia32_maxph128_mask"
| "__builtin_ia32_maxph256_mask"
| "__builtin_ia32_minph128_mask"
| "__builtin_ia32_minph256_mask" => {
let mut new_args = args.to_vec();
let arg3_type = gcc_func.get_param_type(2);
let vector_type = arg3_type.dyncast_vector().expect("vector type");
@ -182,7 +223,19 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
| "__builtin_ia32_vplzcntd_128_mask"
| "__builtin_ia32_vplzcntq_512_mask"
| "__builtin_ia32_vplzcntq_256_mask"
| "__builtin_ia32_vplzcntq_128_mask" => {
| "__builtin_ia32_vplzcntq_128_mask"
| "__builtin_ia32_cvtqq2pd128_mask"
| "__builtin_ia32_cvtqq2pd256_mask"
| "__builtin_ia32_cvtqq2ps256_mask"
| "__builtin_ia32_cvtuqq2pd128_mask"
| "__builtin_ia32_cvtuqq2pd256_mask"
| "__builtin_ia32_cvtuqq2ps256_mask"
| "__builtin_ia32_vcvtw2ph128_mask"
| "__builtin_ia32_vcvtw2ph256_mask"
| "__builtin_ia32_vcvtuw2ph128_mask"
| "__builtin_ia32_vcvtuw2ph256_mask"
| "__builtin_ia32_vcvtdq2ph256_mask"
| "__builtin_ia32_vcvtudq2ph256_mask" => {
let mut new_args = args.to_vec();
// Remove last arg as it doesn't seem to be used in GCC and is always false.
new_args.pop();
@ -281,7 +334,11 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
new_args.push(last_arg);
args = new_args.into();
}
"__builtin_ia32_vfmaddsubps512_mask" | "__builtin_ia32_vfmaddsubpd512_mask" => {
"__builtin_ia32_vfmaddsubps512_mask"
| "__builtin_ia32_vfmaddsubpd512_mask"
| "__builtin_ia32_cmpsh_mask_round"
| "__builtin_ia32_vfmaddph512_mask"
| "__builtin_ia32_vfmaddsubph512_mask" => {
let mut new_args = args.to_vec();
let last_arg = new_args.pop().expect("last arg");
let arg4_type = gcc_func.get_param_type(3);
@ -304,9 +361,8 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
| "__builtin_ia32_vpermi2varpd128_mask"
| "__builtin_ia32_vpmadd52huq512_mask"
| "__builtin_ia32_vpmadd52luq512_mask"
| "__builtin_ia32_vpmadd52huq256_mask"
| "__builtin_ia32_vpmadd52luq256_mask"
| "__builtin_ia32_vpmadd52huq128_mask" => {
| "__builtin_ia32_vfmaddsubph128_mask"
| "__builtin_ia32_vfmaddsubph256_mask" => {
let mut new_args = args.to_vec();
let arg4_type = gcc_func.get_param_type(3);
let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
@ -355,7 +411,14 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
args = vec![new_args[1], new_args[0], new_args[2], minus_one].into();
}
"__builtin_ia32_xrstor" | "__builtin_ia32_xsavec" => {
"__builtin_ia32_xrstor"
| "__builtin_ia32_xrstor64"
| "__builtin_ia32_xsavec"
| "__builtin_ia32_xsavec64"
| "__builtin_ia32_xsave"
| "__builtin_ia32_xsave64"
| "__builtin_ia32_xsaveopt"
| "__builtin_ia32_xsaveopt64" => {
let new_args = args.to_vec();
let thirty_two = builder.context.new_rvalue_from_int(new_args[1].get_type(), 32);
let arg2 = new_args[1] << thirty_two | new_args[2];
@ -378,11 +441,76 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
);
args = vec![arg.get_address(None)].into();
}
"__builtin_ia32_cvtqq2pd512_mask"
| "__builtin_ia32_cvtqq2ps512_mask"
| "__builtin_ia32_cvtuqq2pd512_mask"
| "__builtin_ia32_cvtuqq2ps512_mask"
| "__builtin_ia32_sqrtph512_mask_round"
| "__builtin_ia32_vcvtw2ph512_mask_round"
| "__builtin_ia32_vcvtuw2ph512_mask_round"
| "__builtin_ia32_vcvtdq2ph512_mask_round"
| "__builtin_ia32_vcvtudq2ph512_mask_round"
| "__builtin_ia32_vcvtqq2ph512_mask_round"
| "__builtin_ia32_vcvtuqq2ph512_mask_round" => {
let mut old_args = args.to_vec();
let mut new_args = vec![];
new_args.push(old_args.swap_remove(0));
let arg2_type = gcc_func.get_param_type(1);
let vector_type = arg2_type.dyncast_vector().expect("vector type");
let zero = builder.context.new_rvalue_zero(vector_type.get_element_type());
let num_units = vector_type.get_num_units();
let first_arg =
builder.context.new_rvalue_from_vector(None, arg2_type, &vec![zero; num_units]);
new_args.push(first_arg);
let arg3_type = gcc_func.get_param_type(2);
let minus_one = builder.context.new_rvalue_from_int(arg3_type, -1);
new_args.push(minus_one);
new_args.push(old_args.swap_remove(0));
args = new_args.into();
}
"__builtin_ia32_addph512_mask_round"
| "__builtin_ia32_subph512_mask_round"
| "__builtin_ia32_mulph512_mask_round"
| "__builtin_ia32_divph512_mask_round"
| "__builtin_ia32_maxph512_mask_round"
| "__builtin_ia32_minph512_mask_round" => {
let mut new_args = args.to_vec();
let last_arg = new_args.pop().expect("last arg");
let arg3_type = gcc_func.get_param_type(2);
let vector_type = arg3_type.dyncast_vector().expect("vector type");
let zero = builder.context.new_rvalue_zero(vector_type.get_element_type());
let num_units = vector_type.get_num_units();
let first_arg =
builder.context.new_rvalue_from_vector(None, arg3_type, &vec![zero; num_units]);
new_args.push(first_arg);
let arg4_type = gcc_func.get_param_type(3);
let minus_one = builder.context.new_rvalue_from_int(arg4_type, -1);
new_args.push(minus_one);
new_args.push(last_arg);
args = new_args.into();
}
// NOTE: the LLVM intrinsics receive 3 floats, but the GCC builtin requires 3 vectors.
"__builtin_ia32_vfmaddsh3_mask" => {
let new_args = args.to_vec();
let arg1_type = gcc_func.get_param_type(0);
let arg2_type = gcc_func.get_param_type(1);
let arg3_type = gcc_func.get_param_type(2);
let arg4_type = gcc_func.get_param_type(3);
let a = builder.context.new_rvalue_from_vector(None, arg1_type, &[new_args[0]; 8]);
let b = builder.context.new_rvalue_from_vector(None, arg2_type, &[new_args[1]; 8]);
let c = builder.context.new_rvalue_from_vector(None, arg3_type, &[new_args[2]; 8]);
let arg4 = builder.context.new_rvalue_from_int(arg4_type, -1);
args = vec![a, b, c, arg4, new_args[3]].into();
}
_ => (),
}
} else {
match func_name {
"__builtin_ia32_rndscaless_mask_round" | "__builtin_ia32_rndscalesd_mask_round" => {
"__builtin_ia32_rndscaless_mask_round"
| "__builtin_ia32_rndscalesd_mask_round"
| "__builtin_ia32_reducesh_mask_round" => {
let new_args = args.to_vec();
let arg3_type = gcc_func.get_param_type(2);
let arg3 = builder.context.new_cast(None, new_args[4], arg3_type);
@ -390,7 +518,7 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
let arg4 = builder.context.new_bitcast(None, new_args[2], arg4_type);
args = vec![new_args[0], new_args[1], arg3, arg4, new_args[3], new_args[5]].into();
}
// NOTE: the LLVM intrinsic receives 3 floats, but the GCC builtin requires 3 vectors.
// NOTE: the LLVM intrinsics receive 3 floats, but the GCC builtin requires 3 vectors.
// FIXME: the intrinsics like _mm_mask_fmadd_sd should probably directly call the GCC
// intrinsic to avoid this.
"__builtin_ia32_vfmaddss3_round" => {
@ -473,6 +601,52 @@ pub fn adjust_intrinsic_arguments<'a, 'b, 'gcc, 'tcx>(
let new_args = args.to_vec();
args = vec![new_args[1], new_args[0], new_args[2]].into();
}
"__builtin_ia32_rangesd128_mask_round"
| "__builtin_ia32_rangess128_mask_round"
| "__builtin_ia32_reducesd_mask_round"
| "__builtin_ia32_reducess_mask_round" => {
let new_args = args.to_vec();
args = vec![
new_args[0],
new_args[1],
new_args[4],
new_args[2],
new_args[3],
new_args[5],
]
.into();
}
"__builtin_ia32_rndscalesh_mask_round" => {
let new_args = args.to_vec();
args = vec![
new_args[0],
new_args[1],
new_args[4],
new_args[2],
new_args[3],
new_args[5],
]
.into();
}
"fma" => {
let mut new_args = args.to_vec();
new_args[0] = builder.context.new_cast(None, new_args[0], builder.double_type);
new_args[1] = builder.context.new_cast(None, new_args[1], builder.double_type);
new_args[2] = builder.context.new_cast(None, new_args[2], builder.double_type);
args = new_args.into();
}
"__builtin_ia32_sqrtsh_mask_round"
| "__builtin_ia32_vcvtss2sh_mask_round"
| "__builtin_ia32_vcvtsd2sh_mask_round"
| "__builtin_ia32_vcvtsh2ss_mask_round"
| "__builtin_ia32_vcvtsh2sd_mask_round"
| "__builtin_ia32_rcpsh_mask"
| "__builtin_ia32_rsqrtsh_mask" => {
// The first two arguments are inverted, so swap them.
let mut new_args = args.to_vec();
new_args.swap(0, 1);
args = new_args.into();
}
_ => (),
}
}
@ -489,7 +663,9 @@ pub fn adjust_intrinsic_return_value<'a, 'gcc, 'tcx>(
orig_args: &[RValue<'gcc>],
) -> RValue<'gcc> {
match func_name {
"__builtin_ia32_vfmaddss3_round" | "__builtin_ia32_vfmaddsd3_round" => {
"__builtin_ia32_vfmaddss3_round"
| "__builtin_ia32_vfmaddsd3_round"
| "__builtin_ia32_vfmaddsh3_mask" => {
#[cfg(feature = "master")]
{
let zero = builder.context.new_rvalue_zero(builder.int_type);
@ -546,6 +722,10 @@ pub fn adjust_intrinsic_return_value<'a, 'gcc, 'tcx>(
success_variable.to_rvalue(),
]);
}
"fma" => {
let f16_type = builder.context.new_c_type(CType::Float16);
return_value = builder.context.new_cast(None, return_value, f16_type);
}
_ => (),
}
@ -779,7 +959,9 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
"llvm.x86.avx512.mask.cmp.b.256" => "__builtin_ia32_cmpb256_mask",
"llvm.x86.avx512.mask.cmp.b.128" => "__builtin_ia32_cmpb128_mask",
"llvm.x86.xrstor" => "__builtin_ia32_xrstor",
"llvm.x86.xrstor64" => "__builtin_ia32_xrstor64",
"llvm.x86.xsavec" => "__builtin_ia32_xsavec",
"llvm.x86.xsavec64" => "__builtin_ia32_xsavec64",
"llvm.x86.addcarry.32" => "__builtin_ia32_addcarryx_u32",
"llvm.x86.subborrow.32" => "__builtin_ia32_sbb_u32",
"llvm.x86.avx512.mask.compress.store.w.512" => "__builtin_ia32_compressstoreuhi512_mask",
@ -968,9 +1150,9 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
"llvm.x86.avx512.dbpsadbw.128" => "__builtin_ia32_dbpsadbw128_mask",
"llvm.x86.avx512.vpmadd52h.uq.512" => "__builtin_ia32_vpmadd52huq512_mask",
"llvm.x86.avx512.vpmadd52l.uq.512" => "__builtin_ia32_vpmadd52luq512_mask",
"llvm.x86.avx512.vpmadd52h.uq.256" => "__builtin_ia32_vpmadd52huq256_mask",
"llvm.x86.avx512.vpmadd52l.uq.256" => "__builtin_ia32_vpmadd52luq256_mask",
"llvm.x86.avx512.vpmadd52h.uq.128" => "__builtin_ia32_vpmadd52huq128_mask",
"llvm.x86.avx512.vpmadd52h.uq.256" => "__builtin_ia32_vpmadd52huq256",
"llvm.x86.avx512.vpmadd52l.uq.256" => "__builtin_ia32_vpmadd52luq256",
"llvm.x86.avx512.vpmadd52h.uq.128" => "__builtin_ia32_vpmadd52huq128",
"llvm.x86.avx512.vpdpwssd.512" => "__builtin_ia32_vpdpwssd_v16si",
"llvm.x86.avx512.vpdpwssd.256" => "__builtin_ia32_vpdpwssd_v8si",
"llvm.x86.avx512.vpdpwssd.128" => "__builtin_ia32_vpdpwssd_v4si",
@ -983,6 +1165,180 @@ pub fn intrinsic<'gcc, 'tcx>(name: &str, cx: &CodegenCx<'gcc, 'tcx>) -> Function
"llvm.x86.avx512.vpdpbusds.512" => "__builtin_ia32_vpdpbusds_v16si",
"llvm.x86.avx512.vpdpbusds.256" => "__builtin_ia32_vpdpbusds_v8si",
"llvm.x86.avx512.vpdpbusds.128" => "__builtin_ia32_vpdpbusds_v4si",
"llvm.x86.xsave" => "__builtin_ia32_xsave",
"llvm.x86.xsave64" => "__builtin_ia32_xsave64",
"llvm.x86.xsaveopt" => "__builtin_ia32_xsaveopt",
"llvm.x86.xsaveopt64" => "__builtin_ia32_xsaveopt64",
"llvm.x86.avx512.mask.loadu.w.512" => "__builtin_ia32_loaddquhi512_mask",
"llvm.x86.avx512.mask.loadu.b.512" => "__builtin_ia32_loaddquqi512_mask",
"llvm.x86.avx512.mask.loadu.w.256" => "__builtin_ia32_loaddquhi256_mask",
"llvm.x86.avx512.mask.loadu.b.256" => "__builtin_ia32_loaddquqi256_mask",
"llvm.x86.avx512.mask.loadu.w.128" => "__builtin_ia32_loaddquhi128_mask",
"llvm.x86.avx512.mask.loadu.b.128" => "__builtin_ia32_loaddquqi128_mask",
"llvm.x86.avx512.mask.storeu.w.512" => "__builtin_ia32_storedquhi512_mask",
"llvm.x86.avx512.mask.storeu.b.512" => "__builtin_ia32_storedquqi512_mask",
"llvm.x86.avx512.mask.storeu.w.256" => "__builtin_ia32_storedquhi256_mask",
"llvm.x86.avx512.mask.storeu.b.256" => "__builtin_ia32_storedquqi256_mask",
"llvm.x86.avx512.mask.storeu.w.128" => "__builtin_ia32_storedquhi128_mask",
"llvm.x86.avx512.mask.storeu.b.128" => "__builtin_ia32_storedquqi128_mask",
"llvm.x86.avx512.mask.expand.load.w.512" => "__builtin_ia32_expandloadhi512_mask",
"llvm.x86.avx512.mask.expand.load.w.256" => "__builtin_ia32_expandloadhi256_mask",
"llvm.x86.avx512.mask.expand.load.w.128" => "__builtin_ia32_expandloadhi128_mask",
"llvm.x86.avx512.mask.expand.load.b.512" => "__builtin_ia32_expandloadqi512_mask",
"llvm.x86.avx512.mask.expand.load.b.256" => "__builtin_ia32_expandloadqi256_mask",
"llvm.x86.avx512.mask.expand.load.b.128" => "__builtin_ia32_expandloadqi128_mask",
"llvm.x86.avx512.sitofp.round.v8f64.v8i64" => "__builtin_ia32_cvtqq2pd512_mask",
"llvm.x86.avx512.sitofp.round.v2f64.v2i64" => "__builtin_ia32_cvtqq2pd128_mask",
"llvm.x86.avx512.sitofp.round.v4f64.v4i64" => "__builtin_ia32_cvtqq2pd256_mask",
"llvm.x86.avx512.sitofp.round.v8f32.v8i64" => "__builtin_ia32_cvtqq2ps512_mask",
"llvm.x86.avx512.sitofp.round.v4f32.v4i64" => "__builtin_ia32_cvtqq2ps256_mask",
"llvm.x86.avx512.uitofp.round.v8f64.v8u64" => "__builtin_ia32_cvtuqq2pd512_mask",
"llvm.x86.avx512.uitofp.round.v2f64.v2u64" => "__builtin_ia32_cvtuqq2pd128_mask",
"llvm.x86.avx512.uitofp.round.v4f64.v4u64" => "__builtin_ia32_cvtuqq2pd256_mask",
"llvm.x86.avx512.uitofp.round.v8f32.v8u64" => "__builtin_ia32_cvtuqq2ps512_mask",
"llvm.x86.avx512.uitofp.round.v4f32.v4u64" => "__builtin_ia32_cvtuqq2ps256_mask",
"llvm.x86.avx512.mask.reduce.pd.512" => "__builtin_ia32_reducepd512_mask_round",
"llvm.x86.avx512.mask.reduce.ps.512" => "__builtin_ia32_reduceps512_mask_round",
"llvm.x86.avx512.mask.reduce.sd" => "__builtin_ia32_reducesd_mask_round",
"llvm.x86.avx512.mask.reduce.ss" => "__builtin_ia32_reducess_mask_round",
"llvm.x86.avx512.mask.loadu.d.256" => "__builtin_ia32_loaddqusi256_mask",
"llvm.x86.avx512.mask.loadu.q.256" => "__builtin_ia32_loaddqudi256_mask",
"llvm.x86.avx512.mask.loadu.ps.256" => "__builtin_ia32_loadups256_mask",
"llvm.x86.avx512.mask.loadu.pd.256" => "__builtin_ia32_loadupd256_mask",
"llvm.x86.avx512.mask.loadu.d.128" => "__builtin_ia32_loaddqusi128_mask",
"llvm.x86.avx512.mask.loadu.q.128" => "__builtin_ia32_loaddqudi128_mask",
"llvm.x86.avx512.mask.loadu.ps.128" => "__builtin_ia32_loadups128_mask",
"llvm.x86.avx512.mask.loadu.pd.128" => "__builtin_ia32_loadupd128_mask",
"llvm.x86.avx512.mask.load.d.512" => "__builtin_ia32_movdqa32load512_mask",
"llvm.x86.avx512.mask.load.q.512" => "__builtin_ia32_movdqa64load512_mask",
"llvm.x86.avx512.mask.load.ps.512" => "__builtin_ia32_loadaps512_mask",
"llvm.x86.avx512.mask.load.pd.512" => "__builtin_ia32_loadapd512_mask",
"llvm.x86.avx512.mask.load.d.256" => "__builtin_ia32_movdqa32load256_mask",
"llvm.x86.avx512.mask.load.q.256" => "__builtin_ia32_movdqa64load256_mask",
"llvm.x86.avx512fp16.mask.cmp.sh" => "__builtin_ia32_cmpsh_mask_round",
"llvm.x86.avx512fp16.vcomi.sh" => "__builtin_ia32_cmpsh_mask_round",
"llvm.x86.avx512fp16.add.ph.512" => "__builtin_ia32_addph512_mask_round",
"llvm.x86.avx512fp16.sub.ph.512" => "__builtin_ia32_subph512_mask_round",
"llvm.x86.avx512fp16.mul.ph.512" => "__builtin_ia32_mulph512_mask_round",
"llvm.x86.avx512fp16.div.ph.512" => "__builtin_ia32_divph512_mask_round",
"llvm.x86.avx512fp16.mask.vfmul.cph.512" => "__builtin_ia32_vfmulcph512_mask_round",
"llvm.x86.avx512fp16.mask.vfmul.csh" => "__builtin_ia32_vfmulcsh_mask_round",
"llvm.x86.avx512fp16.mask.vfcmul.cph.512" => "__builtin_ia32_vfcmulcph512_mask_round",
"llvm.x86.avx512fp16.mask.vfcmul.csh" => "__builtin_ia32_vfcmulcsh_mask_round",
"llvm.x86.avx512fp16.mask.vfmadd.cph.512" => "__builtin_ia32_vfmaddcph512_mask3_round",
"llvm.x86.avx512fp16.maskz.vfmadd.cph.512" => "__builtin_ia32_vfmaddcph512_maskz_round",
"llvm.x86.avx512fp16.mask.vfmadd.csh" => "__builtin_ia32_vfmaddcsh_mask3_round",
"llvm.x86.avx512fp16.maskz.vfmadd.csh" => "__builtin_ia32_vfmaddcsh_maskz_round",
"llvm.x86.avx512fp16.mask.vfcmadd.cph.512" => "__builtin_ia32_vfcmaddcph512_mask3_round",
"llvm.x86.avx512fp16.maskz.vfcmadd.cph.512" => "__builtin_ia32_vfcmaddcph512_maskz_round",
"llvm.x86.avx512fp16.mask.vfcmadd.csh" => "__builtin_ia32_vfcmaddcsh_mask3_round",
"llvm.x86.avx512fp16.maskz.vfcmadd.csh" => "__builtin_ia32_vfcmaddcsh_maskz_round",
"llvm.x86.avx512fp16.vfmadd.ph.512" => "__builtin_ia32_vfmaddph512_mask",
"llvm.x86.avx512fp16.vcvtsi642sh" => "__builtin_ia32_vcvtsi2sh64_round",
"llvm.x86.avx512fp16.vcvtusi642sh" => "__builtin_ia32_vcvtusi2sh64_round",
"llvm.x86.avx512fp16.vcvtsh2si64" => "__builtin_ia32_vcvtsh2si64_round",
"llvm.x86.avx512fp16.vcvtsh2usi64" => "__builtin_ia32_vcvtsh2usi64_round",
"llvm.x86.avx512fp16.vcvttsh2si64" => "__builtin_ia32_vcvttsh2si64_round",
"llvm.x86.avx512fp16.vcvttsh2usi64" => "__builtin_ia32_vcvttsh2usi64_round",
"llvm.x86.avx512.mask.load.ps.256" => "__builtin_ia32_loadaps256_mask",
"llvm.x86.avx512.mask.load.pd.256" => "__builtin_ia32_loadapd256_mask",
"llvm.x86.avx512.mask.load.d.128" => "__builtin_ia32_movdqa32load128_mask",
"llvm.x86.avx512.mask.load.q.128" => "__builtin_ia32_movdqa64load128_mask",
"llvm.x86.avx512.mask.load.ps.128" => "__builtin_ia32_loadaps128_mask",
"llvm.x86.avx512.mask.load.pd.128" => "__builtin_ia32_loadapd128_mask",
"llvm.x86.avx512.mask.storeu.d.256" => "__builtin_ia32_storedqusi256_mask",
"llvm.x86.avx512.mask.storeu.q.256" => "__builtin_ia32_storedqudi256_mask",
"llvm.x86.avx512.mask.storeu.ps.256" => "__builtin_ia32_storeups256_mask",
"llvm.x86.avx512.mask.storeu.pd.256" => "__builtin_ia32_storeupd256_mask",
"llvm.x86.avx512.mask.storeu.d.128" => "__builtin_ia32_storedqusi128_mask",
"llvm.x86.avx512.mask.storeu.q.128" => "__builtin_ia32_storedqudi128_mask",
"llvm.x86.avx512.mask.storeu.ps.128" => "__builtin_ia32_storeups128_mask",
"llvm.x86.avx512.mask.storeu.pd.128" => "__builtin_ia32_storeupd128_mask",
"llvm.x86.avx512.mask.store.d.512" => "__builtin_ia32_movdqa32store512_mask",
"llvm.x86.avx512.mask.store.q.512" => "__builtin_ia32_movdqa64store512_mask",
"llvm.x86.avx512.mask.store.ps.512" => "__builtin_ia32_storeaps512_mask",
"llvm.x86.avx512.mask.store.pd.512" => "__builtin_ia32_storeapd512_mask",
"llvm.x86.avx512.mask.store.d.256" => "__builtin_ia32_movdqa32store256_mask",
"llvm.x86.avx512.mask.store.q.256" => "__builtin_ia32_movdqa64store256_mask",
"llvm.x86.avx512.mask.store.ps.256" => "__builtin_ia32_storeaps256_mask",
"llvm.x86.avx512.mask.store.pd.256" => "__builtin_ia32_storeapd256_mask",
"llvm.x86.avx512.mask.store.d.128" => "__builtin_ia32_movdqa32store128_mask",
"llvm.x86.avx512.mask.store.q.128" => "__builtin_ia32_movdqa64store128_mask",
"llvm.x86.avx512.mask.store.ps.128" => "__builtin_ia32_storeaps128_mask",
"llvm.x86.avx512.mask.store.pd.128" => "__builtin_ia32_storeapd128_mask",
"llvm.x86.avx512fp16.vfmadd.f16" => "__builtin_ia32_vfmaddsh3_mask",
"llvm.x86.avx512fp16.vfmaddsub.ph.128" => "__builtin_ia32_vfmaddsubph128_mask",
"llvm.x86.avx512fp16.vfmaddsub.ph.256" => "__builtin_ia32_vfmaddsubph256_mask",
"llvm.x86.avx512fp16.vfmaddsub.ph.512" => "__builtin_ia32_vfmaddsubph512_mask",
"llvm.x86.avx512fp16.sqrt.ph.512" => "__builtin_ia32_sqrtph512_mask_round",
"llvm.x86.avx512fp16.mask.sqrt.sh" => "__builtin_ia32_sqrtsh_mask_round",
"llvm.x86.avx512fp16.max.ph.128" => "__builtin_ia32_maxph128_mask",
"llvm.x86.avx512fp16.max.ph.256" => "__builtin_ia32_maxph256_mask",
"llvm.x86.avx512fp16.max.ph.512" => "__builtin_ia32_maxph512_mask_round",
"llvm.x86.avx512fp16.min.ph.128" => "__builtin_ia32_minph128_mask",
"llvm.x86.avx512fp16.min.ph.256" => "__builtin_ia32_minph256_mask",
"llvm.x86.avx512fp16.min.ph.512" => "__builtin_ia32_minph512_mask_round",
"llvm.x86.avx512fp16.mask.getexp.sh" => "__builtin_ia32_getexpsh_mask_round",
"llvm.x86.avx512fp16.mask.rndscale.ph.128" => "__builtin_ia32_rndscaleph128_mask",
"llvm.x86.avx512fp16.mask.rndscale.ph.256" => "__builtin_ia32_rndscaleph256_mask",
"llvm.x86.avx512fp16.mask.rndscale.ph.512" => "__builtin_ia32_rndscaleph512_mask_round",
"llvm.x86.avx512fp16.mask.scalef.ph.512" => "__builtin_ia32_scalefph512_mask_round",
"llvm.x86.avx512fp16.mask.reduce.ph.512" => "__builtin_ia32_reduceph512_mask_round",
"llvm.x86.avx512fp16.mask.reduce.sh" => "__builtin_ia32_reducesh_mask_round",
"llvm.x86.avx512.sitofp.round.v8f16.v8i16" => "__builtin_ia32_vcvtw2ph128_mask",
"llvm.x86.avx512.sitofp.round.v16f16.v16i16" => "__builtin_ia32_vcvtw2ph256_mask",
"llvm.x86.avx512.sitofp.round.v32f16.v32i16" => "__builtin_ia32_vcvtw2ph512_mask_round",
"llvm.x86.avx512.uitofp.round.v8f16.v8u16" => "__builtin_ia32_vcvtuw2ph128_mask",
"llvm.x86.avx512.uitofp.round.v16f16.v16u16" => "__builtin_ia32_vcvtuw2ph256_mask",
"llvm.x86.avx512.uitofp.round.v32f16.v32u16" => "__builtin_ia32_vcvtuw2ph512_mask_round",
"llvm.x86.avx512.sitofp.round.v8f16.v8i32" => "__builtin_ia32_vcvtdq2ph256_mask",
"llvm.x86.avx512.sitofp.round.v16f16.v16i32" => "__builtin_ia32_vcvtdq2ph512_mask_round",
"llvm.x86.avx512fp16.vcvtsi2sh" => "__builtin_ia32_vcvtsi2sh32_round",
"llvm.x86.avx512.uitofp.round.v8f16.v8u32" => "__builtin_ia32_vcvtudq2ph256_mask",
"llvm.x86.avx512.uitofp.round.v16f16.v16u32" => "__builtin_ia32_vcvtudq2ph512_mask_round",
"llvm.x86.avx512fp16.vcvtusi2sh" => "__builtin_ia32_vcvtusi2sh32_round",
"llvm.x86.avx512.sitofp.round.v8f16.v8i64" => "__builtin_ia32_vcvtqq2ph512_mask_round",
"llvm.x86.avx512.uitofp.round.v8f16.v8u64" => "__builtin_ia32_vcvtuqq2ph512_mask_round",
"llvm.x86.avx512fp16.mask.vcvtps2phx.512" => "__builtin_ia32_vcvtps2phx512_mask_round",
"llvm.x86.avx512fp16.mask.vcvtpd2ph.512" => "__builtin_ia32_vcvtpd2ph512_mask_round",
"llvm.x86.avx512fp16.mask.vcvtph2uw.512" => "__builtin_ia32_vcvtph2uw512_mask_round",
"llvm.x86.avx512fp16.mask.vcvttph2w.512" => "__builtin_ia32_vcvttph2w512_mask_round",
"llvm.x86.avx512fp16.mask.vcvttph2uw.512" => "__builtin_ia32_vcvttph2uw512_mask_round",
"llvm.x86.avx512fp16.mask.vcvtph2dq.512" => "__builtin_ia32_vcvtph2dq512_mask_round",
"llvm.x86.avx512fp16.vcvtsh2si32" => "__builtin_ia32_vcvtsh2si32_round",
"llvm.x86.avx512fp16.mask.vcvtph2udq.512" => "__builtin_ia32_vcvtph2udq512_mask_round",
"llvm.x86.avx512fp16.vcvtsh2usi32" => "__builtin_ia32_vcvtsh2usi32_round",
"llvm.x86.avx512fp16.mask.vcvttph2dq.512" => "__builtin_ia32_vcvttph2dq512_mask_round",
"llvm.x86.avx512fp16.vcvttsh2si32" => "__builtin_ia32_vcvttsh2si32_round",
"llvm.x86.avx512fp16.mask.vcvttph2udq.512" => "__builtin_ia32_vcvttph2udq512_mask_round",
"llvm.x86.avx512fp16.vcvttsh2usi32" => "__builtin_ia32_vcvttsh2usi32_round",
"llvm.x86.avx512fp16.mask.vcvtph2qq.512" => "__builtin_ia32_vcvtph2qq512_mask_round",
"llvm.x86.avx512fp16.mask.vcvtph2uqq.512" => "__builtin_ia32_vcvtph2uqq512_mask_round",
"llvm.x86.avx512fp16.mask.vcvttph2qq.512" => "__builtin_ia32_vcvttph2qq512_mask_round",
"llvm.x86.avx512fp16.mask.vcvttph2uqq.512" => "__builtin_ia32_vcvttph2uqq512_mask_round",
"llvm.x86.avx512fp16.mask.vcvtph2psx.512" => "__builtin_ia32_vcvtph2psx512_mask_round",
"llvm.x86.avx512fp16.mask.vcvtph2pd.512" => "__builtin_ia32_vcvtph2pd512_mask_round",
"llvm.x86.avx512fp16.mask.vfcmadd.cph.256" => "__builtin_ia32_vfcmaddcph256_mask3",
"llvm.x86.avx512fp16.mask.vfmadd.cph.256" => "__builtin_ia32_vfmaddcph256_mask3",
"llvm.x86.avx512fp16.mask.vfcmadd.cph.128" => "__builtin_ia32_vfcmaddcph128_mask3",
"llvm.x86.avx512fp16.mask.vfmadd.cph.128" => "__builtin_ia32_vfmaddcph128_mask3",
// TODO: support the tile builtins:
"llvm.x86.ldtilecfg" => "__builtin_trap",
"llvm.x86.sttilecfg" => "__builtin_trap",
"llvm.x86.tileloadd64" => "__builtin_trap",
"llvm.x86.tilerelease" => "__builtin_trap",
"llvm.x86.tilestored64" => "__builtin_trap",
"llvm.x86.tileloaddt164" => "__builtin_trap",
"llvm.x86.tilezero" => "__builtin_trap",
"llvm.x86.tdpbf16ps" => "__builtin_trap",
"llvm.x86.tdpbssd" => "__builtin_trap",
"llvm.x86.tdpbsud" => "__builtin_trap",
"llvm.x86.tdpbusd" => "__builtin_trap",
"llvm.x86.tdpbuud" => "__builtin_trap",
"llvm.x86.tdpfp16ps" => "__builtin_trap",
"llvm.x86.tcmmimfp16ps" => "__builtin_trap",
"llvm.x86.tcmmrlfp16ps" => "__builtin_trap",
// NOTE: this file is generated by https://github.com/GuillaumeGomez/llvmint/blob/master/generate_list.py
_ => include!("archs.rs"),

21
compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
View file

@ -127,20 +127,13 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
// https://github.com/rust-lang/rust-clippy/issues/12497
// and leave `else if use_integer_compare` to be placed "as is".
#[allow(clippy::suspicious_else_formatting)]
let llval = match name {
let value = match name {
_ if simple.is_some() => {
// FIXME(antoyo): remove this cast when the API supports function.
let func = unsafe {
std::mem::transmute::<Function<'gcc>, RValue<'gcc>>(simple.expect("simple"))
};
self.call(
self.type_void(),
None,
None,
let func = simple.expect("simple function");
self.cx.context.new_call(
self.location,
func,
&args.iter().map(|arg| arg.immediate()).collect::<Vec<_>>(),
None,
None,
)
}
sym::likely => self.expect(args[0].immediate(), true),
@ -383,7 +376,7 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
_ if name_str.starts_with("simd_") => {
match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) {
Ok(llval) => llval,
Ok(value) => value,
Err(()) => return Ok(()),
}
}
@ -396,9 +389,9 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tc
if let PassMode::Cast { cast: ref ty, .. } = fn_abi.ret.mode {
let ptr_llty = self.type_ptr_to(ty.gcc_type(self));
let ptr = self.pointercast(result.val.llval, ptr_llty);
self.store(llval, ptr, result.val.align);
self.store(value, ptr, result.val.align);
} else {
OperandRef::from_immediate_or_packed_pair(self, llval, result.layout)
OperandRef::from_immediate_or_packed_pair(self, value, result.layout)
.val
.store(self, result);
}

55
compiler/rustc_codegen_gcc/src/intrinsic/simd.rs
View file

@ -198,7 +198,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
bx.context.new_bitcast(None, shuffled, v_type)
};
if name == sym::simd_bswap || name == sym::simd_bitreverse {
if matches!(name, sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop) {
require!(
bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer,
InvalidMonomorphization::UnsupportedOperation { span, name, in_ty, in_elem }
@ -209,6 +209,22 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
return Ok(simd_bswap(bx, args[0].immediate()));
}
let simd_ctpop = |bx: &mut Builder<'a, 'gcc, 'tcx>, vector: RValue<'gcc>| -> RValue<'gcc> {
let mut vector_elements = vec![];
let elem_ty = bx.element_type(llret_ty);
for i in 0..in_len {
let index = bx.context.new_rvalue_from_long(bx.ulong_type, i as i64);
let element = bx.extract_element(vector, index).to_rvalue();
let result = bx.context.new_cast(None, bx.pop_count(element), elem_ty);
vector_elements.push(result);
}
bx.context.new_rvalue_from_vector(None, llret_ty, &vector_elements)
};
if name == sym::simd_ctpop {
return Ok(simd_ctpop(bx, args[0].immediate()));
}
// We use a different algorithm from non-vector bitreverse to take advantage of most
// processors' vector shuffle units. It works like this:
// 1. Generate pre-reversed low and high nibbles as a vector.
@ -718,11 +734,12 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
return Err(());
}};
}
let (elem_ty_str, elem_ty) = if let ty::Float(ref f) = *in_elem.kind() {
let (elem_ty_str, elem_ty, cast_type) = if let ty::Float(ref f) = *in_elem.kind() {
let elem_ty = bx.cx.type_float_from_ty(*f);
match f.bit_width() {
32 => ("f", elem_ty),
64 => ("", elem_ty),
16 => ("", elem_ty, Some(bx.cx.double_type)),
32 => ("f", elem_ty, None),
64 => ("", elem_ty, None),
_ => {
return_error!(InvalidMonomorphization::FloatingPointVector {
span,
@ -758,10 +775,7 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
_ => return_error!(InvalidMonomorphization::UnrecognizedIntrinsic { span, name }),
};
let builtin_name = format!("{}{}", intr_name, elem_ty_str);
let funcs = bx.cx.functions.borrow();
let function = funcs
.get(&builtin_name)
.unwrap_or_else(|| panic!("unable to find builtin function {}", builtin_name));
let function = bx.context.get_builtin_function(builtin_name);
// TODO(antoyo): add platform-specific behavior here for architectures that have these
// intrinsics as instructions (for instance, gpus)
@ -769,17 +783,28 @@ pub fn generic_simd_intrinsic<'a, 'gcc, 'tcx>(
for i in 0..in_len {
let index = bx.context.new_rvalue_from_long(bx.ulong_type, i as i64);
// we have to treat fpowi specially, since fpowi's second argument is always an i32
let arguments = if name == sym::simd_fpowi {
vec![
let mut arguments = vec![];
if name == sym::simd_fpowi {
arguments = vec![
bx.extract_element(args[0].immediate(), index).to_rvalue(),
args[1].immediate(),
]
];
} else {
args.iter()
.map(|arg| bx.extract_element(arg.immediate(), index).to_rvalue())
.collect()
for arg in args {
let mut element = bx.extract_element(arg.immediate(), index).to_rvalue();
// FIXME: it would probably be better to not have casts here and use the proper
// instructions.
if let Some(typ) = cast_type {
element = bx.context.new_cast(None, element, typ);
}
arguments.push(element);
}
};
vector_elements.push(bx.context.new_call(None, *function, &arguments));
let mut result = bx.context.new_call(None, function, &arguments);
if cast_type.is_some() {
result = bx.context.new_cast(None, result, elem_ty);
}
vector_elements.push(result);
}
let c = bx.context.new_rvalue_from_vector(None, vec_ty, &vector_elements);
Ok(c)

2
compiler/rustc_codegen_gcc/src/lib.rs
View file

@ -363,7 +363,7 @@ impl Deref for SyncContext {
unsafe impl Send for SyncContext {}
// FIXME(antoyo): that shouldn't be Sync. Parallel compilation is currently disabled with "-Zno-parallel-llvm".
// TODO: disable it here by returing false in CodegenBackend::supports_parallel().
// TODO: disable it here by returning false in CodegenBackend::supports_parallel().
unsafe impl Sync for SyncContext {}
impl WriteBackendMethods for GccCodegenBackend {

2
compiler/rustc_codegen_gcc/src/mono_item.rs
View file

@ -37,7 +37,7 @@ impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> {
let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL);
let global = self.define_global(symbol_name, gcc_type, is_tls, attrs.link_section);
#[cfg(feature = "master")]
global.add_string_attribute(VarAttribute::Visibility(base::visibility_to_gcc(visibility)));
global.add_attribute(VarAttribute::Visibility(base::visibility_to_gcc(visibility)));
// TODO(antoyo): set linkage.
self.instances.borrow_mut().insert(instance, global);

4
compiler/rustc_codegen_gcc/tests/failing-ice-tests.txt
View file

@ -34,3 +34,7 @@ tests/ui/sepcomp/sepcomp-unwind.rs
tests/ui/extern/issue-64655-extern-rust-must-allow-unwind.rs
tests/ui/extern/issue-64655-allow-unwind-when-calling-panic-directly.rs
tests/ui/unwind-no-uwtable.rs
tests/ui/delegation/fn-header.rs
tests/ui/simd/intrinsic/generic-arithmetic-pass.rs
tests/ui/simd/masked-load-store.rs
tests/ui/rfcs/rfc-2632-const-trait-impl/effects/minicore.rs

26
compiler/rustc_codegen_gcc/tests/failing-ui-tests.txt
View file

@ -95,3 +95,29 @@ tests/ui/simd/intrinsic/generic-arithmetic-pass.rs
tests/ui/backtrace/backtrace.rs
tests/ui/lifetimes/tail-expr-lock-poisoning.rs
tests/ui/runtime/rt-explody-panic-payloads.rs
tests/ui/codegen/equal-pointers-unequal/as-cast/function.rs
tests/ui/codegen/equal-pointers-unequal/as-cast/basic.rs
tests/ui/codegen/equal-pointers-unequal/as-cast/inline1.rs
tests/ui/codegen/equal-pointers-unequal/as-cast/print.rs
tests/ui/codegen/equal-pointers-unequal/as-cast/inline2.rs
tests/ui/codegen/equal-pointers-unequal/as-cast/print3.rs
tests/ui/codegen/equal-pointers-unequal/as-cast/segfault.rs
tests/ui/codegen/equal-pointers-unequal/exposed-provenance/function.rs
tests/ui/codegen/equal-pointers-unequal/exposed-provenance/basic.rs
tests/ui/codegen/equal-pointers-unequal/as-cast/zero.rs
tests/ui/codegen/equal-pointers-unequal/exposed-provenance/inline1.rs
tests/ui/codegen/equal-pointers-unequal/exposed-provenance/print.rs
tests/ui/codegen/equal-pointers-unequal/exposed-provenance/print3.rs
tests/ui/codegen/equal-pointers-unequal/exposed-provenance/inline2.rs
tests/ui/codegen/equal-pointers-unequal/exposed-provenance/segfault.rs
tests/ui/codegen/equal-pointers-unequal/exposed-provenance/zero.rs
tests/ui/codegen/equal-pointers-unequal/strict-provenance/basic.rs
tests/ui/codegen/equal-pointers-unequal/strict-provenance/function.rs
tests/ui/codegen/equal-pointers-unequal/strict-provenance/print.rs
tests/ui/codegen/equal-pointers-unequal/strict-provenance/inline1.rs
tests/ui/codegen/equal-pointers-unequal/strict-provenance/print3.rs
tests/ui/codegen/equal-pointers-unequal/strict-provenance/inline2.rs
tests/ui/codegen/equal-pointers-unequal/strict-provenance/segfault.rs
tests/ui/codegen/equal-pointers-unequal/strict-provenance/zero.rs
tests/ui/sanitizer/cfi-sized-associated-ty.rs
tests/ui/sanitizer/cfi-can-reveal-opaques.rs

1
compiler/rustc_codegen_gcc/tests/run/array.rs
View file

@ -31,6 +31,7 @@ impl Copy for i32 {}
impl Copy for u8 {}
impl Copy for i8 {}
impl Copy for i16 {}
impl<T: ?Sized> Copy for *mut T {}
#[lang = "receiver"]
trait Receiver {

1
compiler/rustc_codegen_gcc/tests/run/closure.rs
View file

@ -33,6 +33,7 @@ impl Copy for i32 {}
impl Copy for u32 {}
impl Copy for u8 {}
impl Copy for i8 {}
impl<T: ?Sized> Copy for *mut T {}
#[lang = "receiver"]
trait Receiver {

1
compiler/rustc_codegen_gcc/tests/run/condition.rs
View file

@ -34,6 +34,7 @@ impl Copy for i16 {}
impl Copy for char {}
impl Copy for i8 {}
impl Copy for u8 {}
impl<T: ?Sized> Copy for *mut T {}
#[lang = "receiver"]
trait Receiver {

1
compiler/rustc_codegen_gcc/tests/run/fun_ptr.rs
View file

@ -28,6 +28,7 @@ impl Copy for i32 {}
impl Copy for u8 {}
impl Copy for i8 {}
impl Copy for i16 {}
impl<T: ?Sized> Copy for *mut T {}
#[lang = "receiver"]
trait Receiver {

1
compiler/rustc_codegen_gcc/tests/run/ptr_cast.rs
View file

@ -28,6 +28,7 @@ impl Copy for i32 {}
impl Copy for u8 {}
impl Copy for i8 {}
impl Copy for i16 {}
impl<T: ?Sized> Copy for *mut T {}
#[lang = "receiver"]
trait Receiver {

1
compiler/rustc_codegen_gcc/tests/run/slice.rs
View file

@ -26,6 +26,7 @@ impl Copy for isize {}
impl Copy for usize {}
impl Copy for i32 {}
impl Copy for u32 {}
impl<T: ?Sized> Copy for *mut T {}
#[lang = "receiver"]
trait Receiver {

1
compiler/rustc_codegen_gcc/tests/run/static.rs
View file

@ -34,6 +34,7 @@ trait Copy {
}
impl Copy for isize {}
impl<T: ?Sized> Copy for *mut T {}
#[lang = "receiver"]
trait Receiver {

10
compiler/rustc_codegen_gcc/tools/generate_intrinsics.py
View file

@ -45,7 +45,7 @@ def convert_to_string(content):
return content
def extract_instrinsics_from_llvm(llvm_path, intrinsics):
def extract_intrinsics_from_llvm(llvm_path, intrinsics):
command = ["llvm-tblgen", "llvm/IR/Intrinsics.td"]
cwd = os.path.join(llvm_path, "llvm/include")
print("=> Running command `{}` from `{}`".format(command, cwd))
@ -88,7 +88,7 @@ def append_translation(json_data, p, array):
append_intrinsic(array, content[1], content[3])
def extract_instrinsics_from_llvmint(llvmint, intrinsics):
def extract_intrinsics_from_llvmint(llvmint, intrinsics):
archs = [
"AMDGPU",
"aarch64",
@ -152,9 +152,9 @@ def update_intrinsics(llvm_path, llvmint, llvmint2):
intrinsics_llvmint = {}
all_intrinsics = {}
extract_instrinsics_from_llvm(llvm_path, intrinsics_llvm)
extract_instrinsics_from_llvmint(llvmint, intrinsics_llvmint)
extract_instrinsics_from_llvmint(llvmint2, intrinsics_llvmint)
extract_intrinsics_from_llvm(llvm_path, intrinsics_llvm)
extract_intrinsics_from_llvmint(llvmint, intrinsics_llvmint)
extract_intrinsics_from_llvmint(llvmint2, intrinsics_llvmint)
intrinsics = {}
# We give priority to translations from LLVM over the ones from llvmint.

22
compiler/rustc_codegen_llvm/src/allocator.rs
View file

@ -77,18 +77,20 @@ pub(crate) unsafe fn codegen(
// __rust_alloc_error_handler_should_panic
let name = OomStrategy::SYMBOL;
let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8);
if tcx.sess.default_hidden_visibility() {
llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden);
}
llvm::LLVMRustSetVisibility(
ll_g,
llvm::Visibility::from_generic(tcx.sess.default_visibility()),
);
let val = tcx.sess.opts.unstable_opts.oom.should_panic();
let llval = llvm::LLVMConstInt(i8, val as u64, False);
llvm::LLVMSetInitializer(ll_g, llval);
let name = NO_ALLOC_SHIM_IS_UNSTABLE;
let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8);
if tcx.sess.default_hidden_visibility() {
llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden);
}
llvm::LLVMRustSetVisibility(
ll_g,
llvm::Visibility::from_generic(tcx.sess.default_visibility()),
);
let llval = llvm::LLVMConstInt(i8, 0, False);
llvm::LLVMSetInitializer(ll_g, llval);
}
@ -132,9 +134,11 @@ fn create_wrapper_function(
None
};
if tcx.sess.default_hidden_visibility() {
llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden);
}
llvm::LLVMRustSetVisibility(
llfn,
llvm::Visibility::from_generic(tcx.sess.default_visibility()),
);
if tcx.sess.must_emit_unwind_tables() {
let uwtable =
attributes::uwtable_attr(llcx, tcx.sess.opts.unstable_opts.use_sync_unwind);

6
compiler/rustc_codegen_llvm/src/asm.rs
View file

@ -708,6 +708,9 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) ->
S390x(S390xInlineAsmRegClass::reg) => "r",
S390x(S390xInlineAsmRegClass::reg_addr) => "a",
S390x(S390xInlineAsmRegClass::freg) => "f",
S390x(S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg) => {
unreachable!("clobber-only")
}
Msp430(Msp430InlineAsmRegClass::reg) => "r",
M68k(M68kInlineAsmRegClass::reg) => "r",
M68k(M68kInlineAsmRegClass::reg_addr) => "a",
@ -866,6 +869,9 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'
Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
S390x(S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr) => cx.type_i32(),
S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
S390x(S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg) => {
unreachable!("clobber-only")
}
Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(),
M68k(M68kInlineAsmRegClass::reg) => cx.type_i32(),
M68k(M68kInlineAsmRegClass::reg_addr) => cx.type_i32(),

2
compiler/rustc_codegen_llvm/src/back/archive.rs
View file

@ -123,7 +123,7 @@ fn get_llvm_object_symbols(
llvm::LLVMRustGetSymbols(
buf.as_ptr(),
buf.len(),
std::ptr::addr_of_mut!(*state) as *mut c_void,
(&raw mut *state) as *mut c_void,
callback,
error_callback,
)

13
compiler/rustc_codegen_llvm/src/declare.rs
View file

@ -22,6 +22,7 @@ use tracing::debug;
use crate::abi::{FnAbi, FnAbiLlvmExt};
use crate::context::CodegenCx;
use crate::llvm::AttributePlace::Function;
use crate::llvm::Visibility;
use crate::type_::Type;
use crate::value::Value;
use crate::{attributes, llvm};
@ -84,11 +85,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
fn_type: &'ll Type,
) -> &'ll Value {
// Declare C ABI functions with the visibility used by C by default.
let visibility = if self.tcx.sess.default_hidden_visibility() {
llvm::Visibility::Hidden
} else {
llvm::Visibility::Default
};
let visibility = Visibility::from_generic(self.tcx.sess.default_visibility());
declare_raw_fn(self, name, llvm::CCallConv, unnamed, visibility, fn_type)
}
@ -107,11 +104,7 @@ impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> {
unnamed: llvm::UnnamedAddr,
fn_type: &'ll Type,
) -> &'ll Value {
let visibility = if self.tcx.sess.default_hidden_visibility() {
llvm::Visibility::Hidden
} else {
llvm::Visibility::Default
};
let visibility = Visibility::from_generic(self.tcx.sess.default_visibility());
declare_raw_fn(self, name, callconv, unnamed, visibility, fn_type)
}

4
compiler/rustc_codegen_llvm/src/lib.rs
View file

@ -167,7 +167,7 @@ impl WriteBackendMethods for LlvmCodegenBackend {
fn print_pass_timings(&self) {
unsafe {
let mut size = 0;
let cstr = llvm::LLVMRustPrintPassTimings(std::ptr::addr_of_mut!(size));
let cstr = llvm::LLVMRustPrintPassTimings(&raw mut size);
if cstr.is_null() {
println!("failed to get pass timings");
} else {
@ -180,7 +180,7 @@ impl WriteBackendMethods for LlvmCodegenBackend {
fn print_statistics(&self) {
unsafe {
let mut size = 0;
let cstr = llvm::LLVMRustPrintStatistics(std::ptr::addr_of_mut!(size));
let cstr = llvm::LLVMRustPrintStatistics(&raw mut size);
if cstr.is_null() {
println!("failed to get pass stats");
} else {

11
compiler/rustc_codegen_llvm/src/llvm/ffi.rs
View file

@ -4,6 +4,7 @@
use std::marker::PhantomData;
use libc::{c_char, c_int, c_uint, c_ulonglong, c_void, size_t};
use rustc_target::spec::SymbolVisibility;
use super::RustString;
use super::debuginfo::{
@ -133,6 +134,16 @@ pub enum Visibility {
Protected = 2,
}
impl Visibility {
pub fn from_generic(visibility: SymbolVisibility) -> Self {
match visibility {
SymbolVisibility::Hidden => Visibility::Hidden,
SymbolVisibility::Protected => Visibility::Protected,
SymbolVisibility::Interposable => Visibility::Default,
}
}
}
/// LLVMUnnamedAddr
#[repr(C)]
pub enum UnnamedAddr {

11
compiler/rustc_codegen_llvm/src/llvm_util.rs
View file

@ -264,6 +264,10 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea
("riscv32" | "riscv64", "unaligned-scalar-mem") if get_version().0 == 18 => {
Some(LLVMFeature::new("fast-unaligned-access"))
}
// Filter out features that are not supported by the current LLVM version
("riscv32" | "riscv64", "zaamo") if get_version().0 < 19 => None,
("riscv32" | "riscv64", "zabha") if get_version().0 < 19 => None,
("riscv32" | "riscv64", "zalrsc") if get_version().0 < 19 => None,
// Enable the evex512 target feature if an avx512 target feature is enabled.
("x86", s) if s.starts_with("avx512") => {
Some(LLVMFeature::with_dependency(s, TargetFeatureFoldStrength::EnableOnly("evex512")))
@ -478,7 +482,7 @@ pub(crate) fn print(req: &PrintRequest, mut out: &mut String, sess: &Session) {
&tm,
cpu_cstring.as_ptr(),
callback,
std::ptr::addr_of_mut!(out) as *mut c_void,
(&raw mut out) as *mut c_void,
);
}
}
@ -536,6 +540,11 @@ pub(crate) fn global_llvm_features(
// -Ctarget-cpu=native
match sess.opts.cg.target_cpu {
Some(ref s) if s == "native" => {
// We have already figured out the actual CPU name with `LLVMRustGetHostCPUName` and set
// that for LLVM, so the features implied by that CPU name will be available everywhere.
// However, that is not sufficient: e.g. `skylake` alone is not sufficient to tell if
// some of the instructions are available or not. So we have to also explicitly ask for
// the exact set of features available on the host, and enable all of them.
let features_string = unsafe {
let ptr = llvm::LLVMGetHostCPUFeatures();
let features_string = if !ptr.is_null() {

7
compiler/rustc_codegen_ssa/Cargo.toml
View file

@ -8,7 +8,7 @@ edition = "2021"
ar_archive_writer = "0.4.2"
arrayvec = { version = "0.7", default-features = false }
bitflags = "2.4.1"
cc = "=1.0.105" # FIXME(cc): pinned to keep support for VS2013
cc = "1.1.23"
either = "1.5.0"
itertools = "0.12"
jobserver = "0.1.28"
@ -34,12 +34,13 @@ rustc_session = { path = "../rustc_session" }
rustc_span = { path = "../rustc_span" }
rustc_symbol_mangling = { path = "../rustc_symbol_mangling" }
rustc_target = { path = "../rustc_target" }
rustc_trait_selection = { path = "../rustc_trait_selection" }
rustc_type_ir = { path = "../rustc_type_ir" }
serde_json = "1.0.59"
smallvec = { version = "1.8.1", features = ["union", "may_dangle"] }
tempfile = "3.2"
thin-vec = "0.2.12"
thorin-dwp = "0.7"
thorin-dwp = "0.8"
tracing = "0.1"
wasm-encoder = "0.216.0"
# tidy-alphabetical-end
@ -55,5 +56,5 @@ default-features = false
features = ["read_core", "elf", "macho", "pe", "xcoff", "unaligned", "archive", "write", "wasm"]
[target.'cfg(windows)'.dependencies.windows]
version = "0.52.0"
version = "0.57.0"
features = ["Win32_Globalization"]

169
compiler/rustc_codegen_ssa/src/back/link.rs
View file

@ -40,7 +40,7 @@ use rustc_target::spec::crt_objects::CrtObjects;
use rustc_target::spec::{
Cc, LinkOutputKind, LinkSelfContainedComponents, LinkSelfContainedDefault, LinkerFeatures,
LinkerFlavor, LinkerFlavorCli, Lld, PanicStrategy, RelocModel, RelroLevel, SanitizerSet,
SplitDebuginfo,
SplitDebuginfo, current_apple_deployment_target,
};
use tempfile::Builder as TempFileBuilder;
use tracing::{debug, info, warn};
@ -2405,6 +2405,8 @@ fn add_order_independent_options(
// Take care of the flavors and CLI options requesting the `lld` linker.
add_lld_args(cmd, sess, flavor, self_contained_components);
add_apple_link_args(cmd, sess, flavor);
let apple_sdk_root = add_apple_sdk(cmd, sess, flavor);
add_link_script(cmd, sess, tmpdir, crate_type);
@ -2488,7 +2490,7 @@ fn add_order_independent_options(
}
}
cmd.set_output_kind(link_output_kind, out_filename);
cmd.set_output_kind(link_output_kind, crate_type, out_filename);
add_relro_args(cmd, sess);
@ -2957,6 +2959,169 @@ pub(crate) fn are_upstream_rust_objects_already_included(sess: &Session) -> bool
}
}
/// We need to communicate five things to the linker on Apple/Darwin targets:
/// - The architecture.
/// - The operating system (and that it's an Apple platform).
/// - The environment / ABI.
/// - The deployment target.
/// - The SDK version.
fn add_apple_link_args(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) {
if !sess.target.is_like_osx {
return;
}
let LinkerFlavor::Darwin(cc, _) = flavor else {
return;
};
// `sess.target.arch` (`target_arch`) is not detailed enough.
let llvm_arch = sess.target.llvm_target.split_once('-').expect("LLVM target must have arch").0;
let target_os = &*sess.target.os;
let target_abi = &*sess.target.abi;
// The architecture name to forward to the linker.
//
// Supported architecture names can be found in the source:
// https://github.com/apple-oss-distributions/ld64/blob/ld64-951.9/src/abstraction/MachOFileAbstraction.hpp#L578-L648
//
// Intentially verbose to ensure that the list always matches correctly
// with the list in the source above.
let ld64_arch = match llvm_arch {
"armv7k" => "armv7k",
"armv7s" => "armv7s",
"arm64" => "arm64",
"arm64e" => "arm64e",
"arm64_32" => "arm64_32",
// ld64 doesn't understand i686, so fall back to i386 instead.
//
// Same story when linking with cc, since that ends up invoking ld64.
"i386" | "i686" => "i386",
"x86_64" => "x86_64",
"x86_64h" => "x86_64h",
_ => bug!("unsupported architecture in Apple target: {}", sess.target.llvm_target),
};
if cc == Cc::No {
// From the man page for ld64 (`man ld`):
// > The linker accepts universal (multiple-architecture) input files,
// > but always creates a "thin" (single-architecture), standard
// > Mach-O output file. The architecture for the output file is
// > specified using the -arch option.
//
// The linker has heuristics to determine the desired architecture,
// but to be safe, and to avoid a warning, we set the architecture
// explicitly.
cmd.link_args(&["-arch", ld64_arch]);
// Man page says that ld64 supports the following platform names:
// > - macos
// > - ios
// > - tvos
// > - watchos
// > - bridgeos
// > - visionos
// > - xros
// > - mac-catalyst
// > - ios-simulator
// > - tvos-simulator
// > - watchos-simulator
// > - visionos-simulator
// > - xros-simulator
// > - driverkit
let platform_name = match (target_os, target_abi) {
(os, "") => os,
("ios", "macabi") => "mac-catalyst",
("ios", "sim") => "ios-simulator",
("tvos", "sim") => "tvos-simulator",
("watchos", "sim") => "watchos-simulator",
("visionos", "sim") => "visionos-simulator",
_ => bug!("invalid OS/ABI combination for Apple target: {target_os}, {target_abi}"),
};
let (major, minor, patch) = current_apple_deployment_target(&sess.target);
let min_version = format!("{major}.{minor}.{patch}");
// The SDK version is used at runtime when compiling with a newer SDK / version of Xcode:
// - By dyld to give extra warnings and errors, see e.g.:
// <https://github.com/apple-oss-distributions/dyld/blob/dyld-1165.3/common/MachOFile.cpp#L3029>
// <https://github.com/apple-oss-distributions/dyld/blob/dyld-1165.3/common/MachOFile.cpp#L3738-L3857>
// - By system frameworks to change certain behaviour. For example, the default value of
// `-[NSView wantsBestResolutionOpenGLSurface]` is `YES` when the SDK version is >= 10.15.
// <https://developer.apple.com/documentation/appkit/nsview/1414938-wantsbestresolutionopenglsurface?language=objc>
//
// We do not currently know the actual SDK version though, so we have a few options:
// 1. Use the minimum version supported by rustc.
// 2. Use the same as the deployment target.
// 3. Use an arbitary recent version.
// 4. Omit the version.
//
// The first option is too low / too conservative, and means that users will not get the
// same behaviour from a binary compiled with rustc as with one compiled by clang.
//
// The second option is similarly conservative, and also wrong since if the user specified a
// higher deployment target than the SDK they're compiling/linking with, the runtime might
// make invalid assumptions about the capabilities of the binary.
//
// The third option requires that `rustc` is periodically kept up to date with Apple's SDK
// version, and is also wrong for similar reasons as above.
//
// The fourth option is bad because while `ld`, `otool`, `vtool` and such understand it to
// mean "absent" or `n/a`, dyld doesn't actually understand it, and will end up interpreting
// it as 0.0, which is again too low/conservative.
//
// Currently, we lie about the SDK version, and choose the second option.
//
// FIXME(madsmtm): Parse the SDK version from the SDK root instead.
// <https://github.com/rust-lang/rust/issues/129432>
let sdk_version = &*min_version;
// From the man page for ld64 (`man ld`):
// > This is set to indicate the platform, oldest supported version of
// > that platform that output is to be used on, and the SDK that the
// > output was built against.
//
// Like with `-arch`, the linker can figure out the platform versions
// itself from the binaries being linked, but to be safe, we specify
// the desired versions here explicitly.
cmd.link_args(&["-platform_version", platform_name, &*min_version, sdk_version]);
} else {
// cc == Cc::Yes
//
// We'd _like_ to use `-target` everywhere, since that can uniquely
// communicate all the required details except for the SDK version
// (which is read by Clang itself from the SDKROOT), but that doesn't
// work on GCC, and since we don't know whether the `cc` compiler is
// Clang, GCC, or something else, we fall back to other options that
// also work on GCC when compiling for macOS.
//
// Targets other than macOS are ill-supported by GCC (it doesn't even
// support e.g. `-miphoneos-version-min`), so in those cases we can
// fairly safely use `-target`. See also the following, where it is
// made explicit that the recommendation by LLVM developers is to use
// `-target`: <https://github.com/llvm/llvm-project/issues/88271>
if target_os == "macos" {
// `-arch` communicates the architecture.
//
// CC forwards the `-arch` to the linker, so we use the same value
// here intentionally.
cmd.cc_args(&["-arch", ld64_arch]);
// The presence of `-mmacosx-version-min` makes CC default to
// macOS, and it sets the deployment target.
let (major, minor, patch) = current_apple_deployment_target(&sess.target);
// Intentionally pass this as a single argument, Clang doesn't
// seem to like it otherwise.
cmd.cc_arg(&format!("-mmacosx-version-min={major}.{minor}.{patch}"));
// macOS has no environment, so with these two, we've told CC the
// four desired parameters.
//
// We avoid `-m32`/`-m64`, as this is already encoded by `-arch`.
} else {
cmd.cc_args(&["-target", &sess.target.llvm_target]);
}
}
}
fn add_apple_sdk(cmd: &mut dyn Linker, sess: &Session, flavor: LinkerFlavor) -> Option<PathBuf> {
let arch = &sess.target.arch;
let os = &sess.target.os;

83
compiler/rustc_codegen_ssa/src/back/linker.rs
View file

@ -275,7 +275,12 @@ pub(crate) trait Linker {
fn is_cc(&self) -> bool {
false
}
fn set_output_kind(&mut self, output_kind: LinkOutputKind, out_filename: &Path);
fn set_output_kind(
&mut self,
output_kind: LinkOutputKind,
crate_type: CrateType,
out_filename: &Path,
);
fn link_dylib_by_name(&mut self, _name: &str, _verbatim: bool, _as_needed: bool) {
bug!("dylib linked with unsupported linker")
}
@ -396,7 +401,7 @@ impl<'a> GccLinker<'a> {
]);
}
fn build_dylib(&mut self, out_filename: &Path) {
fn build_dylib(&mut self, crate_type: CrateType, out_filename: &Path) {
// On mac we need to tell the linker to let this library be rpathed
if self.sess.target.is_like_osx {
if !self.is_ld {
@ -427,7 +432,7 @@ impl<'a> GccLinker<'a> {
let mut out_implib = OsString::from("--out-implib=");
out_implib.push(out_filename.with_file_name(implib_name));
self.link_arg(out_implib);
} else {
} else if crate_type == CrateType::Dylib {
// When dylibs are linked by a full path this value will get into `DT_NEEDED`
// instead of the full path, so the library can be later found in some other
// location than that specific path.
@ -474,7 +479,12 @@ impl<'a> Linker for GccLinker<'a> {
!self.is_ld
}
fn set_output_kind(&mut self, output_kind: LinkOutputKind, out_filename: &Path) {
fn set_output_kind(
&mut self,
output_kind: LinkOutputKind,
crate_type: CrateType,
out_filename: &Path,
) {
match output_kind {
LinkOutputKind::DynamicNoPicExe => {
if !self.is_ld && self.is_gnu {
@ -509,10 +519,10 @@ impl<'a> Linker for GccLinker<'a> {
self.link_args(&["-static", "-pie", "--no-dynamic-linker", "-z", "text"]);
}
}
LinkOutputKind::DynamicDylib => self.build_dylib(out_filename),
LinkOutputKind::DynamicDylib => self.build_dylib(crate_type, out_filename),
LinkOutputKind::StaticDylib => {
self.link_or_cc_arg("-static");
self.build_dylib(out_filename);
self.build_dylib(crate_type, out_filename);
}
LinkOutputKind::WasiReactorExe => {
self.link_args(&["--entry", "_initialize"]);
@ -866,7 +876,12 @@ impl<'a> Linker for MsvcLinker<'a> {
&mut self.cmd
}
fn set_output_kind(&mut self, output_kind: LinkOutputKind, out_filename: &Path) {
fn set_output_kind(
&mut self,
output_kind: LinkOutputKind,
_crate_type: CrateType,
out_filename: &Path,
) {
match output_kind {
LinkOutputKind::DynamicNoPicExe
| LinkOutputKind::DynamicPicExe
@ -1124,7 +1139,13 @@ impl<'a> Linker for EmLinker<'a> {
true
}
fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {}
fn set_output_kind(
&mut self,
_output_kind: LinkOutputKind,
_crate_type: CrateType,
_out_filename: &Path,
) {
}
fn link_dylib_by_name(&mut self, name: &str, _verbatim: bool, _as_needed: bool) {
// Emscripten always links statically
@ -1273,7 +1294,12 @@ impl<'a> Linker for WasmLd<'a> {
&mut self.cmd
}
fn set_output_kind(&mut self, output_kind: LinkOutputKind, _out_filename: &Path) {
fn set_output_kind(
&mut self,
output_kind: LinkOutputKind,
_crate_type: CrateType,
_out_filename: &Path,
) {
match output_kind {
LinkOutputKind::DynamicNoPicExe
| LinkOutputKind::DynamicPicExe
@ -1422,7 +1448,13 @@ impl<'a> Linker for L4Bender<'a> {
&mut self.cmd
}
fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {}
fn set_output_kind(
&mut self,
_output_kind: LinkOutputKind,
_crate_type: CrateType,
_out_filename: &Path,
) {
}
fn link_staticlib_by_name(&mut self, name: &str, _verbatim: bool, whole_archive: bool) {
self.hint_static();
@ -1568,7 +1600,12 @@ impl<'a> Linker for AixLinker<'a> {
&mut self.cmd
}
fn set_output_kind(&mut self, output_kind: LinkOutputKind, out_filename: &Path) {
fn set_output_kind(
&mut self,
output_kind: LinkOutputKind,
_crate_type: CrateType,
out_filename: &Path,
) {
match output_kind {
LinkOutputKind::DynamicDylib => {
self.hint_dynamic();
@ -1775,7 +1812,13 @@ impl<'a> Linker for PtxLinker<'a> {
&mut self.cmd
}
fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {}
fn set_output_kind(
&mut self,
_output_kind: LinkOutputKind,
_crate_type: CrateType,
_out_filename: &Path,
) {
}
fn link_staticlib_by_name(&mut self, _name: &str, _verbatim: bool, _whole_archive: bool) {
panic!("staticlibs not supported")
@ -1841,7 +1884,13 @@ impl<'a> Linker for LlbcLinker<'a> {
&mut self.cmd
}
fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {}
fn set_output_kind(
&mut self,
_output_kind: LinkOutputKind,
_crate_type: CrateType,
_out_filename: &Path,
) {
}
fn link_staticlib_by_name(&mut self, _name: &str, _verbatim: bool, _whole_archive: bool) {
panic!("staticlibs not supported")
@ -1912,7 +1961,13 @@ impl<'a> Linker for BpfLinker<'a> {
&mut self.cmd
}
fn set_output_kind(&mut self, _output_kind: LinkOutputKind, _out_filename: &Path) {}
fn set_output_kind(
&mut self,
_output_kind: LinkOutputKind,
_crate_type: CrateType,
_out_filename: &Path,
) {
}
fn link_staticlib_by_name(&mut self, _name: &str, _verbatim: bool, _whole_archive: bool) {
panic!("staticlibs not supported")

10
compiler/rustc_codegen_ssa/src/back/metadata.rs
View file

@ -402,13 +402,17 @@ fn macho_object_build_version_for_target(target: &Target) -> object::write::Mach
let platform =
rustc_target::spec::current_apple_platform(target).expect("unknown Apple target OS");
let min_os = rustc_target::spec::current_apple_deployment_target(target);
let (sdk_major, sdk_minor) =
rustc_target::spec::current_apple_sdk_version(platform).expect("unknown Apple target OS");
let mut build_version = object::write::MachOBuildVersion::default();
build_version.platform = platform;
build_version.minos = pack_version(min_os);
build_version.sdk = pack_version((sdk_major, sdk_minor, 0));
// The version here does not _really_ matter, since it is only used at runtime, and we specify
// it when linking the final binary, so we will omit the version. This is also what LLVM does,
// and the tooling also allows this (and shows the SDK version as `n/a`). Finally, it is the
// semantically correct choice, as the SDK has not influenced the binary generated by rustc at
// this point in time.
build_version.sdk = 0;
build_version
}

59
compiler/rustc_codegen_ssa/src/base.rs
View file

@ -27,6 +27,9 @@ use rustc_session::config::{self, CrateType, EntryFnType, OptLevel, OutputType};
use rustc_span::symbol::sym;
use rustc_span::{DUMMY_SP, Symbol};
use rustc_target::abi::FIRST_VARIANT;
use rustc_trait_selection::infer::at::ToTrace;
use rustc_trait_selection::infer::{BoundRegionConversionTime, TyCtxtInferExt};
use rustc_trait_selection::traits::{ObligationCause, ObligationCtxt};
use tracing::{debug, info};
use crate::assert_module_sources::CguReuse;
@ -101,6 +104,54 @@ pub fn compare_simd_types<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
bx.sext(cmp, ret_ty)
}
/// Codegen takes advantage of the additional assumption, where if the
/// principal trait def id of what's being casted doesn't change,
/// then we don't need to adjust the vtable at all. This
/// corresponds to the fact that `dyn Tr<A>: Unsize<dyn Tr<B>>`
/// requires that `A = B`; we don't allow *upcasting* objects
/// between the same trait with different args. If we, for
/// some reason, were to relax the `Unsize` trait, it could become
/// unsound, so let's validate here that the trait refs are subtypes.
pub fn validate_trivial_unsize<'tcx>(
tcx: TyCtxt<'tcx>,
source_data: &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,
target_data: &'tcx ty::List<ty::PolyExistentialPredicate<'tcx>>,
) -> bool {
match (source_data.principal(), target_data.principal()) {
(Some(hr_source_principal), Some(hr_target_principal)) => {
let infcx = tcx.infer_ctxt().build();
let universe = infcx.universe();
let ocx = ObligationCtxt::new(&infcx);
infcx.enter_forall(hr_target_principal, |target_principal| {
let source_principal = infcx.instantiate_binder_with_fresh_vars(
DUMMY_SP,
BoundRegionConversionTime::HigherRankedType,
hr_source_principal,
);
let Ok(()) = ocx.eq_trace(
&ObligationCause::dummy(),
ty::ParamEnv::reveal_all(),
ToTrace::to_trace(
&ObligationCause::dummy(),
hr_target_principal,
hr_source_principal,
),
target_principal,
source_principal,
) else {
return false;
};
if !ocx.select_all_or_error().is_empty() {
return false;
}
infcx.leak_check(universe, None).is_ok()
})
}
(None, None) => true,
_ => false,
}
}
/// Retrieves the information we are losing (making dynamic) in an unsizing
/// adjustment.
///
@ -133,12 +184,8 @@ fn unsized_info<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
// between the same trait with different args. If we, for
// some reason, were to relax the `Unsize` trait, it could become
// unsound, so let's assert here that the trait refs are *equal*.
//
// We can use `assert_eq` because the binders should have been anonymized,
// and because higher-ranked equality now requires the binders are equal.
debug_assert_eq!(
data_a.principal(),
data_b.principal(),
debug_assert!(
validate_trivial_unsize(cx.tcx(), data_a, data_b),
"NOP unsize vtable changed principal trait ref: {data_a} -> {data_b}"
);

1
compiler/rustc_const_eval/src/check_consts/check.rs
View file

@ -317,7 +317,6 @@ impl<'mir, 'tcx> Checker<'mir, 'tcx> {
{
self.error_emitted = Some(guar);
}
self.check_op_spanned(ops::StaticAccess, span)
}
/// Returns whether this place can possibly escape the evaluation of the current const/static

28
compiler/rustc_const_eval/src/check_consts/ops.rs
View file

@ -16,7 +16,6 @@ use rustc_middle::ty::{
suggest_constraining_type_param,
};
use rustc_middle::util::{CallDesugaringKind, CallKind, call_kind};
use rustc_session::parse::feature_err;
use rustc_span::symbol::sym;
use rustc_span::{BytePos, Pos, Span, Symbol};
use rustc_trait_selection::traits::SelectionContext;
@ -477,33 +476,6 @@ impl<'tcx> NonConstOp<'tcx> for RawPtrToIntCast {
}
}
/// An access to a (non-thread-local) `static`.
#[derive(Debug)]
pub(crate) struct StaticAccess;
impl<'tcx> NonConstOp<'tcx> for StaticAccess {
fn status_in_item(&self, ccx: &ConstCx<'_, 'tcx>) -> Status {
if let hir::ConstContext::Static(_) = ccx.const_kind() {
Status::Allowed
} else {
Status::Unstable(sym::const_refs_to_static)
}
}
#[allow(rustc::untranslatable_diagnostic)] // FIXME: make this translatable
fn build_error(&self, ccx: &ConstCx<'_, 'tcx>, span: Span) -> Diag<'tcx> {
let mut err = feature_err(
&ccx.tcx.sess,
sym::const_refs_to_static,
span,
format!("referencing statics in {}s is unstable", ccx.const_kind(),),
);
err
.note("`static` and `const` variables can refer to other `const` variables. A `const` variable, however, cannot refer to a `static` variable.")
.help("to fix this, the value can be extracted to a `const` and then used.");
err
}
}
/// An access to a thread-local `static`.
#[derive(Debug)]
pub(crate) struct ThreadLocalAccess;

7
compiler/rustc_const_eval/src/const_eval/dummy_machine.rs
View file

@ -6,7 +6,8 @@ use rustc_middle::{bug, span_bug, ty};
use rustc_span::def_id::DefId;
use crate::interpret::{
self, HasStaticRootDefId, ImmTy, Immediate, InterpCx, PointerArithmetic, throw_machine_stop,
self, HasStaticRootDefId, ImmTy, Immediate, InterpCx, PointerArithmetic, interp_ok,
throw_machine_stop,
};
/// Macro for machine-specific `InterpError` without allocation.
@ -79,7 +80,7 @@ impl<'tcx> interpret::Machine<'tcx> for DummyMachine {
throw_machine_stop_str!("can't access mutable globals in ConstProp");
}
Ok(())
interp_ok(())
}
fn find_mir_or_eval_fn(
@ -127,7 +128,7 @@ impl<'tcx> interpret::Machine<'tcx> for DummyMachine {
right: &interpret::ImmTy<'tcx, Self::Provenance>,
) -> interpret::InterpResult<'tcx, ImmTy<'tcx, Self::Provenance>> {
use rustc_middle::mir::BinOp::*;
Ok(match bin_op {
interp_ok(match bin_op {
Eq | Ne | Lt | Le | Gt | Ge => {
// Types can differ, e.g. fn ptrs with different `for`.
assert_eq!(left.layout.abi, right.layout.abi);

51
compiler/rustc_const_eval/src/const_eval/eval_queries.rs
View file

@ -20,7 +20,7 @@ use crate::const_eval::CheckAlignment;
use crate::interpret::{
CtfeValidationMode, GlobalId, Immediate, InternKind, InternResult, InterpCx, InterpError,
InterpResult, MPlaceTy, MemoryKind, OpTy, RefTracking, StackPopCleanup, create_static_alloc,
eval_nullary_intrinsic, intern_const_alloc_recursive, throw_exhaust,
eval_nullary_intrinsic, intern_const_alloc_recursive, interp_ok, throw_exhaust,
};
use crate::{CTRL_C_RECEIVED, errors};
@ -98,19 +98,19 @@ fn eval_body_using_ecx<'tcx, R: InterpretationResult<'tcx>>(
return Err(ecx
.tcx
.dcx()
.emit_err(errors::DanglingPtrInFinal { span: ecx.tcx.span, kind: intern_kind })
.into());
.emit_err(errors::DanglingPtrInFinal { span: ecx.tcx.span, kind: intern_kind }))
.into();
}
Err(InternResult::FoundBadMutablePointer) => {
return Err(ecx
.tcx
.dcx()
.emit_err(errors::MutablePtrInFinal { span: ecx.tcx.span, kind: intern_kind })
.into());
.emit_err(errors::MutablePtrInFinal { span: ecx.tcx.span, kind: intern_kind }))
.into();
}
}
Ok(R::make_result(ret, ecx))
interp_ok(R::make_result(ret, ecx))
}
/// The `InterpCx` is only meant to be used to do field and index projections into constants for
@ -147,7 +147,8 @@ pub fn mk_eval_cx_for_const_val<'tcx>(
ty: Ty<'tcx>,
) -> Option<(CompileTimeInterpCx<'tcx>, OpTy<'tcx>)> {
let ecx = mk_eval_cx_to_read_const_val(tcx.tcx, tcx.span, param_env, CanAccessMutGlobal::No);
let op = ecx.const_val_to_op(val, ty, None).ok()?;
// FIXME: is it a problem to discard the error here?
let op = ecx.const_val_to_op(val, ty, None).discard_err()?;
Some((ecx, op))
}
@ -185,12 +186,16 @@ pub(super) fn op_to_const<'tcx>(
_ => false,
};
let immediate = if force_as_immediate {
match ecx.read_immediate(op) {
match ecx.read_immediate(op).report_err() {
Ok(imm) => Right(imm),
Err(err) if !for_diagnostics => {
panic!("normalization works on validated constants: {err:?}")
Err(err) => {
if for_diagnostics {
// This discard the error, but for diagnostics that's okay.
op.as_mplace_or_imm()
} else {
panic!("normalization works on validated constants: {err:?}")
}
}
_ => op.as_mplace_or_imm(),
}
} else {
op.as_mplace_or_imm()
@ -283,17 +288,19 @@ pub fn eval_to_const_value_raw_provider<'tcx>(
let ty::FnDef(_, args) = ty.kind() else {
bug!("intrinsic with type {:?}", ty);
};
return eval_nullary_intrinsic(tcx, key.param_env, def_id, args).map_err(|error| {
let span = tcx.def_span(def_id);
return eval_nullary_intrinsic(tcx, key.param_env, def_id, args).report_err().map_err(
|error| {
let span = tcx.def_span(def_id);
super::report(
tcx,
error.into_kind(),
span,
|| (span, vec![]),
|span, _| errors::NullaryIntrinsicError { span },
)
});
super::report(
tcx,
error.into_kind(),
span,
|| (span, vec![]),
|span, _| errors::NullaryIntrinsicError { span },
)
},
);
}
tcx.eval_to_allocation_raw(key).map(|val| turn_into_const_value(tcx, val, key))
@ -376,6 +383,7 @@ fn eval_in_interpreter<'tcx, R: InterpretationResult<'tcx>>(
);
let res = ecx.load_mir(cid.instance.def, cid.promoted);
res.and_then(|body| eval_body_using_ecx(&mut ecx, cid, body))
.report_err()
.map_err(|error| report_eval_error(&ecx, cid, error))
}
@ -400,6 +408,7 @@ fn const_validate_mplace<'tcx>(
}
};
ecx.const_validate_operand(&mplace.into(), path, &mut ref_tracking, mode)
.report_err()
// Instead of just reporting the `InterpError` via the usual machinery, we give a more targeted
// error about the validation failure.
.map_err(|error| report_validation_error(&ecx, cid, error, alloc_id))?;

68
compiler/rustc_const_eval/src/const_eval/machine.rs
View file

@ -24,8 +24,8 @@ use crate::fluent_generated as fluent;
use crate::interpret::{
self, AllocId, AllocRange, ConstAllocation, CtfeProvenance, FnArg, Frame, GlobalAlloc, ImmTy,
InterpCx, InterpResult, MPlaceTy, OpTy, Pointer, PointerArithmetic, RangeSet, Scalar,
StackPopCleanup, compile_time_machine, err_ub, throw_exhaust, throw_inval, throw_ub_custom,
throw_unsup, throw_unsup_format,
StackPopCleanup, compile_time_machine, interp_ok, throw_exhaust, throw_inval, throw_ub,
throw_ub_custom, throw_unsup, throw_unsup_format,
};
/// When hitting this many interpreted terminators we emit a deny by default lint
@ -247,7 +247,7 @@ impl<'tcx> CompileTimeInterpCx<'tcx> {
let msg = Symbol::intern(self.read_str(&msg_place)?);
let span = self.find_closest_untracked_caller_location();
let (file, line, col) = self.location_triple_for_span(span);
return Err(ConstEvalErrKind::Panic { msg, file, line, col }.into());
return Err(ConstEvalErrKind::Panic { msg, file, line, col }).into();
} else if self.tcx.is_lang_item(def_id, LangItem::PanicFmt) {
// For panic_fmt, call const_panic_fmt instead.
let const_def_id = self.tcx.require_lang_item(LangItem::ConstPanicFmt, None);
@ -259,16 +259,16 @@ impl<'tcx> CompileTimeInterpCx<'tcx> {
self.cur_span(),
);
return Ok(Some(new_instance));
return interp_ok(Some(new_instance));
} else if self.tcx.is_lang_item(def_id, LangItem::AlignOffset) {
let args = self.copy_fn_args(args);
// For align_offset, we replace the function call if the pointer has no address.
match self.align_offset(instance, &args, dest, ret)? {
ControlFlow::Continue(()) => return Ok(Some(instance)),
ControlFlow::Break(()) => return Ok(None),
ControlFlow::Continue(()) => return interp_ok(Some(instance)),
ControlFlow::Break(()) => return interp_ok(None),
}
}
Ok(Some(instance))
interp_ok(Some(instance))
}
/// `align_offset(ptr, target_align)` needs special handling in const eval, because the pointer
@ -323,25 +323,25 @@ impl<'tcx> CompileTimeInterpCx<'tcx> {
dest,
StackPopCleanup::Goto { ret, unwind: mir::UnwindAction::Unreachable },
)?;
Ok(ControlFlow::Break(()))
interp_ok(ControlFlow::Break(()))
} else {
// Not alignable in const, return `usize::MAX`.
let usize_max = Scalar::from_target_usize(self.target_usize_max(), self);
self.write_scalar(usize_max, dest)?;
self.return_to_block(ret)?;
Ok(ControlFlow::Break(()))
interp_ok(ControlFlow::Break(()))
}
}
Err(_addr) => {
// The pointer has an address, continue with function call.
Ok(ControlFlow::Continue(()))
interp_ok(ControlFlow::Continue(()))
}
}
}
/// See documentation on the `ptr_guaranteed_cmp` intrinsic.
fn guaranteed_cmp(&mut self, a: Scalar, b: Scalar) -> InterpResult<'tcx, u8> {
Ok(match (a, b) {
interp_ok(match (a, b) {
// Comparisons between integers are always known.
(Scalar::Int { .. }, Scalar::Int { .. }) => {
if a == b {
@ -403,8 +403,8 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
instance: ty::InstanceKind<'tcx>,
) -> InterpResult<'tcx, &'tcx mir::Body<'tcx>> {
match instance {
ty::InstanceKind::Item(def) => Ok(ecx.tcx.mir_for_ctfe(def)),
_ => Ok(ecx.tcx.instance_mir(instance)),
ty::InstanceKind::Item(def) => interp_ok(ecx.tcx.mir_for_ctfe(def)),
_ => interp_ok(ecx.tcx.instance_mir(instance)),
}
}
@ -422,7 +422,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
// Replace some functions.
let Some(instance) = ecx.hook_special_const_fn(orig_instance, args, dest, ret)? else {
// Call has already been handled.
return Ok(None);
return interp_ok(None);
};
// Only check non-glue functions
@ -444,14 +444,14 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
// This is a const fn. Call it.
// In case of replacement, we return the *original* instance to make backtraces work out
// (and we hope this does not confuse the FnAbi checks too much).
Ok(Some((ecx.load_mir(instance.def, None)?, orig_instance)))
interp_ok(Some((ecx.load_mir(instance.def, None)?, orig_instance)))
}
fn panic_nounwind(ecx: &mut InterpCx<'tcx, Self>, msg: &str) -> InterpResult<'tcx> {
let msg = Symbol::intern(msg);
let span = ecx.find_closest_untracked_caller_location();
let (file, line, col) = ecx.location_triple_for_span(span);
Err(ConstEvalErrKind::Panic { msg, file, line, col }.into())
Err(ConstEvalErrKind::Panic { msg, file, line, col }).into()
}
fn call_intrinsic(
@ -464,7 +464,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
) -> InterpResult<'tcx, Option<ty::Instance<'tcx>>> {
// Shared intrinsics.
if ecx.eval_intrinsic(instance, args, dest, target)? {
return Ok(None);
return interp_ok(None);
}
let intrinsic_name = ecx.tcx.item_name(instance.def_id());
@ -541,7 +541,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
"intrinsic `{intrinsic_name}` is not supported at compile-time"
);
}
return Ok(Some(ty::Instance {
return interp_ok(Some(ty::Instance {
def: ty::InstanceKind::Item(instance.def_id()),
args: instance.args,
}));
@ -550,7 +550,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
// Intrinsic is done, jump to next block.
ecx.return_to_block(target)?;
Ok(None)
interp_ok(None)
}
fn assert_panic(
@ -581,7 +581,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
}
}
};
Err(ConstEvalErrKind::AssertFailure(err).into())
Err(ConstEvalErrKind::AssertFailure(err)).into()
}
fn binary_ptr_op(
@ -652,7 +652,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
}
}
Ok(())
interp_ok(())
}
#[inline(always)]
@ -670,7 +670,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
if !ecx.recursion_limit.value_within_limit(ecx.stack().len() + 1) {
throw_exhaust!(StackFrameLimitReached)
} else {
Ok(frame)
interp_ok(frame)
}
}
@ -700,22 +700,22 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
if is_write {
// Write access. These are never allowed, but we give a targeted error message.
match alloc.mutability {
Mutability::Not => Err(err_ub!(WriteToReadOnly(alloc_id)).into()),
Mutability::Mut => Err(ConstEvalErrKind::ModifiedGlobal.into()),
Mutability::Not => throw_ub!(WriteToReadOnly(alloc_id)),
Mutability::Mut => Err(ConstEvalErrKind::ModifiedGlobal).into(),
}
} else {
// Read access. These are usually allowed, with some exceptions.
if machine.can_access_mut_global == CanAccessMutGlobal::Yes {
// Machine configuration allows us read from anything (e.g., `static` initializer).
Ok(())
interp_ok(())
} else if alloc.mutability == Mutability::Mut {
// Machine configuration does not allow us to read statics (e.g., `const`
// initializer).
Err(ConstEvalErrKind::ConstAccessesMutGlobal.into())
Err(ConstEvalErrKind::ConstAccessesMutGlobal).into()
} else {
// Immutable global, this read is fine.
assert_eq!(alloc.mutability, Mutability::Not);
Ok(())
interp_ok(())
}
}
}
@ -748,9 +748,9 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
// even when there is interior mutability.)
place.map_provenance(CtfeProvenance::as_shared_ref)
};
Ok(ImmTy::from_immediate(new_place.to_ref(ecx), val.layout))
interp_ok(ImmTy::from_immediate(new_place.to_ref(ecx), val.layout))
} else {
Ok(val.clone())
interp_ok(val.clone())
}
}
@ -763,20 +763,20 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
) -> InterpResult<'tcx> {
if range.size == Size::ZERO {
// Nothing to check.
return Ok(());
return interp_ok(());
}
// Reject writes through immutable pointers.
if immutable {
return Err(ConstEvalErrKind::WriteThroughImmutablePointer.into());
return Err(ConstEvalErrKind::WriteThroughImmutablePointer).into();
}
// Everything else is fine.
Ok(())
interp_ok(())
}
fn before_alloc_read(ecx: &InterpCx<'tcx, Self>, alloc_id: AllocId) -> InterpResult<'tcx> {
// Check if this is the currently evaluated static.
if Some(alloc_id) == ecx.machine.static_root_ids.map(|(id, _)| id) {
return Err(ConstEvalErrKind::RecursiveStatic.into());
return Err(ConstEvalErrKind::RecursiveStatic).into();
}
// If this is another static, make sure we fire off the query to detect cycles.
// But only do that when checks for static recursion are enabled.
@ -788,7 +788,7 @@ impl<'tcx> interpret::Machine<'tcx> for CompileTimeMachine<'tcx> {
ecx.ctfe_query(|tcx| tcx.eval_static_initializer(def_id))?;
}
}
Ok(())
interp_ok(())
}
fn cached_union_data_range<'e>(

20
compiler/rustc_const_eval/src/const_eval/mod.rs
View file

@ -1,13 +1,12 @@
// Not in interpret to make sure we do not use private implementation details
use rustc_middle::mir::interpret::InterpErrorInfo;
use rustc_middle::query::{Key, TyCtxtAt};
use rustc_middle::ty::{self, Ty, TyCtxt};
use rustc_middle::{bug, mir};
use rustc_target::abi::VariantIdx;
use tracing::instrument;
use crate::interpret::{InterpCx, format_interp_error};
use crate::interpret::InterpCx;
mod dummy_machine;
mod error;
@ -33,17 +32,6 @@ pub(crate) enum ValTreeCreationError<'tcx> {
}
pub(crate) type ValTreeCreationResult<'tcx> = Result<ty::ValTree<'tcx>, ValTreeCreationError<'tcx>>;
impl<'tcx> From<InterpErrorInfo<'tcx>> for ValTreeCreationError<'tcx> {
fn from(err: InterpErrorInfo<'tcx>) -> Self {
ty::tls::with(|tcx| {
bug!(
"Unexpected Undefined Behavior error during valtree construction: {}",
format_interp_error(tcx.dcx(), err),
)
})
}
}
#[instrument(skip(tcx), level = "debug")]
pub(crate) fn try_destructure_mir_constant_for_user_output<'tcx>(
tcx: TyCtxtAt<'tcx>,
@ -60,8 +48,8 @@ pub(crate) fn try_destructure_mir_constant_for_user_output<'tcx>(
return None;
}
ty::Adt(def, _) => {
let variant = ecx.read_discriminant(&op).ok()?;
let down = ecx.project_downcast(&op, variant).ok()?;
let variant = ecx.read_discriminant(&op).discard_err()?;
let down = ecx.project_downcast(&op, variant).discard_err()?;
(def.variants()[variant].fields.len(), Some(variant), down)
}
ty::Tuple(args) => (args.len(), None, op),
@ -70,7 +58,7 @@ pub(crate) fn try_destructure_mir_constant_for_user_output<'tcx>(
let fields_iter = (0..field_count)
.map(|i| {
let field_op = ecx.project_field(&down, i).ok()?;
let field_op = ecx.project_field(&down, i).discard_err()?;
let val = op_to_const(&ecx, &field_op, /* for diagnostics */ true);
Some((val, field_op.layout.ty))
})

8
compiler/rustc_const_eval/src/const_eval/valtrees.rs
View file

@ -92,7 +92,7 @@ fn const_to_valtree_inner<'tcx>(
Ok(ty::ValTree::zst())
}
ty::Bool | ty::Int(_) | ty::Uint(_) | ty::Float(_) | ty::Char => {
let val = ecx.read_immediate(place)?;
let val = ecx.read_immediate(place).unwrap();
let val = val.to_scalar_int().unwrap();
*num_nodes += 1;
@ -114,7 +114,7 @@ fn const_to_valtree_inner<'tcx>(
// equality at compile-time (see `ptr_guaranteed_cmp`).
// However we allow those that are just integers in disguise.
// First, get the pointer. Remember it might be wide!
let val = ecx.read_immediate(place)?;
let val = ecx.read_immediate(place).unwrap();
// We could allow wide raw pointers where both sides are integers in the future,
// but for now we reject them.
if matches!(val.layout.abi, Abi::ScalarPair(..)) {
@ -135,7 +135,7 @@ fn const_to_valtree_inner<'tcx>(
ty::FnPtr(..) => Err(ValTreeCreationError::NonSupportedType(ty)),
ty::Ref(_, _, _) => {
let derefd_place = ecx.deref_pointer(place)?;
let derefd_place = ecx.deref_pointer(place).unwrap();
const_to_valtree_inner(ecx, &derefd_place, num_nodes)
}
@ -159,7 +159,7 @@ fn const_to_valtree_inner<'tcx>(
bug!("uninhabited types should have errored and never gotten converted to valtree")
}
let variant = ecx.read_discriminant(place)?;
let variant = ecx.read_discriminant(place).unwrap();
branches(ecx, place, def.variant(variant).fields.len(), def.is_enum().then_some(variant), num_nodes)
}

60
compiler/rustc_const_eval/src/interpret/call.rs
View file

@ -15,8 +15,8 @@ use tracing::{info, instrument, trace};
use super::{
CtfeProvenance, FnVal, ImmTy, InterpCx, InterpResult, MPlaceTy, Machine, OpTy, PlaceTy,
Projectable, Provenance, ReturnAction, Scalar, StackPopCleanup, StackPopInfo, throw_ub,
throw_ub_custom, throw_unsup_format,
Projectable, Provenance, ReturnAction, Scalar, StackPopCleanup, StackPopInfo, interp_ok,
throw_ub, throw_ub_custom, throw_unsup_format,
};
use crate::fluent_generated as fluent;
@ -64,7 +64,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
arg: &FnArg<'tcx, M::Provenance>,
field: usize,
) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> {
Ok(match arg {
interp_ok(match arg {
FnArg::Copy(op) => FnArg::Copy(self.project_field(op, field)?),
FnArg::InPlace(mplace) => FnArg::InPlace(self.project_field(mplace, field)?),
})
@ -97,7 +97,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// another type.
let ty::Adt(def, args) = layout.ty.kind() else {
// Not an ADT, so definitely no NPO.
return Ok(layout);
return interp_ok(layout);
};
let inner = if self.tcx.is_diagnostic_item(sym::Option, def.did()) {
// The wrapped type is the only arg.
@ -111,10 +111,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
} else if rhs.is_1zst() {
lhs
} else {
return Ok(layout); // no NPO
return interp_ok(layout); // no NPO
}
} else {
return Ok(layout); // no NPO
return interp_ok(layout); // no NPO
};
// Check if the inner type is one of the NPO-guaranteed ones.
@ -126,7 +126,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// Stop at NPO types so that we don't miss that attribute in the check below!
def.is_struct() && !is_npo(def)
});
Ok(match inner.ty.kind() {
interp_ok(match inner.ty.kind() {
ty::Ref(..) | ty::FnPtr(..) => {
// Option<&T> behaves like &T, and same for fn()
inner
@ -153,11 +153,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
) -> InterpResult<'tcx, bool> {
// Fast path: equal types are definitely compatible.
if caller.ty == callee.ty {
return Ok(true);
return interp_ok(true);
}
// 1-ZST are compatible with all 1-ZST (and with nothing else).
if caller.is_1zst() || callee.is_1zst() {
return Ok(caller.is_1zst() && callee.is_1zst());
return interp_ok(caller.is_1zst() && callee.is_1zst());
}
// Unfold newtypes and NPO optimizations.
let unfold = |layout: TyAndLayout<'tcx>| {
@ -180,17 +180,17 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
_ => None,
};
if let (Some(caller), Some(callee)) = (thin_pointer(caller), thin_pointer(callee)) {
return Ok(caller == callee);
return interp_ok(caller == callee);
}
// For wide pointers we have to get the pointee type.
let pointee_ty = |ty: Ty<'tcx>| -> InterpResult<'tcx, Option<Ty<'tcx>>> {
// We cannot use `builtin_deref` here since we need to reject `Box<T, MyAlloc>`.
Ok(Some(match ty.kind() {
interp_ok(Some(match ty.kind() {
ty::Ref(_, ty, _) => *ty,
ty::RawPtr(ty, _) => *ty,
// We only accept `Box` with the default allocator.
_ if ty.is_box_global(*self.tcx) => ty.expect_boxed_ty(),
_ => return Ok(None),
_ => return interp_ok(None),
}))
};
if let (Some(caller), Some(callee)) = (pointee_ty(caller.ty)?, pointee_ty(callee.ty)?) {
@ -202,7 +202,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let normalize = |ty| self.tcx.normalize_erasing_regions(self.param_env, ty);
ty.ptr_metadata_ty(*self.tcx, normalize)
};
return Ok(meta_ty(caller) == meta_ty(callee));
return interp_ok(meta_ty(caller) == meta_ty(callee));
}
// Compatible integer types (in particular, usize vs ptr-sized-u32/u64).
@ -217,11 +217,11 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
};
if let (Some(caller), Some(callee)) = (int_ty(caller.ty), int_ty(callee.ty)) {
// This is okay if they are the same integer type.
return Ok(caller == callee);
return interp_ok(caller == callee);
}
// Fall back to exact equality.
Ok(caller == callee)
interp_ok(caller == callee)
}
fn check_argument_compat(
@ -235,13 +235,13 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// Ensure that our checks imply actual ABI compatibility for this concrete call.
// (This can fail e.g. if `#[rustc_nonnull_optimization_guaranteed]` is used incorrectly.)
assert!(caller_abi.eq_abi(callee_abi));
Ok(true)
interp_ok(true)
} else {
trace!(
"check_argument_compat: incompatible ABIs:\ncaller: {:?}\ncallee: {:?}",
caller_abi, callee_abi
);
Ok(false)
interp_ok(false)
}
}
@ -266,7 +266,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if !already_live {
self.storage_live(callee_arg.as_local().unwrap())?;
}
return Ok(());
return interp_ok(());
}
// Find next caller arg.
let Some((caller_arg, caller_abi)) = caller_args.next() else {
@ -308,7 +308,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if let FnArg::InPlace(mplace) = caller_arg {
M::protect_in_place_function_argument(self, mplace)?;
}
Ok(())
interp_ok(())
}
/// The main entry point for creating a new stack frame: performs ABI checks and initializes
@ -536,7 +536,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
unwind,
);
} else {
Ok(())
interp_ok(())
}
}
ty::InstanceKind::VTableShim(..)
@ -561,7 +561,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
unwind,
)?
else {
return Ok(());
return interp_ok(());
};
// Special handling for the closure ABI: untuple the last argument.
@ -572,7 +572,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
trace!("init_fn_call: Will pass last argument by untupling");
Cow::from(
args.iter()
.map(|a| Ok(a.clone()))
.map(|a| interp_ok(a.clone()))
.chain(
(0..untuple_arg.layout().fields.count())
.map(|i| self.fn_arg_field(untuple_arg, i)),
@ -598,7 +598,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// codegen'd / interpreted as virtual calls through the vtable.
ty::InstanceKind::Virtual(def_id, idx) => {
let mut args = args.to_vec();
// We have to implement all "object safe receivers". So we have to go search for a
// We have to implement all "dyn-compatible receivers". So we have to go search for a
// pointer or `dyn Trait` type, but it could be wrapped in newtypes. So recursively
// unwrap those newtypes until we are there.
// An `InPlace` does nothing here, we keep the original receiver intact. We can't
@ -886,27 +886,25 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// this transmute.
res
} else {
Ok(())
interp_ok(())
};
// All right, now it is time to actually pop the frame.
let stack_pop_info = self.pop_stack_frame_raw(unwinding)?;
// Report error from return value copy, if any.
copy_ret_result?;
// An error here takes precedence over the copy error.
let (stack_pop_info, ()) = self.pop_stack_frame_raw(unwinding).and(copy_ret_result)?;
match stack_pop_info.return_action {
ReturnAction::Normal => {}
ReturnAction::NoJump => {
// The hook already did everything.
return Ok(());
return interp_ok(());
}
ReturnAction::NoCleanup => {
// If we are not doing cleanup, also skip everything else.
assert!(self.stack().is_empty(), "only the topmost frame should ever be leaked");
assert!(!unwinding, "tried to skip cleanup during unwinding");
// Skip machine hook.
return Ok(());
return interp_ok(());
}
}
@ -931,7 +929,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
self.stack().is_empty(),
"only the bottommost frame can have StackPopCleanup::Root"
);
Ok(())
interp_ok(())
}
}
}

24
compiler/rustc_const_eval/src/interpret/cast.rs
View file

@ -14,7 +14,8 @@ use tracing::trace;
use super::util::ensure_monomorphic_enough;
use super::{
FnVal, ImmTy, Immediate, InterpCx, Machine, OpTy, PlaceTy, err_inval, throw_ub, throw_ub_custom,
FnVal, ImmTy, Immediate, InterpCx, Machine, OpTy, PlaceTy, err_inval, interp_ok, throw_ub,
throw_ub_custom,
};
use crate::fluent_generated as fluent;
@ -157,7 +158,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
self.copy_op_allow_transmute(src, dest)?;
}
}
Ok(())
interp_ok(())
}
/// Handles 'IntToInt' and 'IntToFloat' casts.
@ -169,7 +170,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
assert!(src.layout.ty.is_integral() || src.layout.ty.is_char() || src.layout.ty.is_bool());
assert!(cast_to.ty.is_floating_point() || cast_to.ty.is_integral() || cast_to.ty.is_char());
Ok(ImmTy::from_scalar(
interp_ok(ImmTy::from_scalar(
self.cast_from_int_like(src.to_scalar(), src.layout, cast_to.ty)?,
cast_to,
))
@ -192,7 +193,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
FloatTy::F64 => self.cast_from_float(src.to_scalar().to_f64()?, cast_to.ty),
FloatTy::F128 => self.cast_from_float(src.to_scalar().to_f128()?, cast_to.ty),
};
Ok(ImmTy::from_scalar(val, cast_to))
interp_ok(ImmTy::from_scalar(val, cast_to))
}
/// Handles 'FnPtrToPtr' and 'PtrToPtr' casts.
@ -206,14 +207,14 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// Handle casting any ptr to raw ptr (might be a fat ptr).
if cast_to.size == src.layout.size {
// Thin or fat pointer that just has the ptr kind of target type changed.
return Ok(ImmTy::from_immediate(**src, cast_to));
return interp_ok(ImmTy::from_immediate(**src, cast_to));
} else {
// Casting the metadata away from a fat ptr.
assert_eq!(src.layout.size, 2 * self.pointer_size());
assert_eq!(cast_to.size, self.pointer_size());
assert!(src.layout.ty.is_unsafe_ptr());
return match **src {
Immediate::ScalarPair(data, _) => Ok(ImmTy::from_scalar(data, cast_to)),
Immediate::ScalarPair(data, _) => interp_ok(ImmTy::from_scalar(data, cast_to)),
Immediate::Scalar(..) => span_bug!(
self.cur_span(),
"{:?} input to a fat-to-thin cast ({} -> {})",
@ -240,7 +241,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
Ok(ptr) => M::expose_ptr(self, ptr)?,
Err(_) => {} // Do nothing, exposing an invalid pointer (`None` provenance) is a NOP.
};
Ok(ImmTy::from_scalar(self.cast_from_int_like(scalar, src.layout, cast_to.ty)?, cast_to))
interp_ok(ImmTy::from_scalar(
self.cast_from_int_like(scalar, src.layout, cast_to.ty)?,
cast_to,
))
}
pub fn pointer_with_exposed_provenance_cast(
@ -258,7 +262,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// Then turn address into pointer.
let ptr = M::ptr_from_addr_cast(self, addr)?;
Ok(ImmTy::from_scalar(Scalar::from_maybe_pointer(ptr, self), cast_to))
interp_ok(ImmTy::from_scalar(Scalar::from_maybe_pointer(ptr, self), cast_to))
}
/// Low-level cast helper function. This works directly on scalars and can take 'int-like' input
@ -280,7 +284,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
_ => span_bug!(self.cur_span(), "invalid int-like cast from {}", src_layout.ty),
};
Ok(match *cast_ty.kind() {
interp_ok(match *cast_ty.kind() {
// int -> int
Int(_) | Uint(_) => {
let size = match *cast_ty.kind() {
@ -505,7 +509,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
self.unsize_into(&src_field, cast_ty_field, &dst_field)?;
}
}
Ok(())
interp_ok(())
}
_ => {
// Do not ICE if we are not monomorphic enough.

19
compiler/rustc_const_eval/src/interpret/discriminant.rs
View file

@ -7,7 +7,8 @@ use rustc_target::abi::{self, TagEncoding, VariantIdx, Variants};
use tracing::{instrument, trace};
use super::{
ImmTy, InterpCx, InterpResult, Machine, Projectable, Scalar, Writeable, err_ub, throw_ub,
ImmTy, InterpCx, InterpResult, Machine, Projectable, Scalar, Writeable, err_ub, interp_ok,
throw_ub,
};
impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
@ -48,7 +49,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if actual_variant != variant_index {
throw_ub!(InvalidNichedEnumVariantWritten { enum_ty: dest.layout().ty });
}
Ok(())
interp_ok(())
}
}
}
@ -89,7 +90,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
throw_ub!(UninhabitedEnumVariantRead(index))
}
}
return Ok(index);
return interp_ok(index);
}
Variants::Multiple { tag, ref tag_encoding, tag_field, .. } => {
(tag, tag_encoding, tag_field)
@ -205,7 +206,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if op.layout().for_variant(self, index).abi.is_uninhabited() {
throw_ub!(UninhabitedEnumVariantRead(index))
}
Ok(index)
interp_ok(index)
}
pub fn discriminant_for_variant(
@ -226,7 +227,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
Scalar::from_uint(variant.as_u32(), discr_layout.size)
}
};
Ok(ImmTy::from_scalar(discr_value, discr_layout))
interp_ok(ImmTy::from_scalar(discr_value, discr_layout))
}
/// Computes how to write the tag of a given variant of enum `ty`:
@ -247,7 +248,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// discriminant is encoded implicitly, so any attempt to write
// the wrong discriminant for a `Single` enum will reliably
// result in UB.
Ok(None)
interp_ok(None)
}
abi::Variants::Multiple {
@ -265,7 +266,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let tag_size = tag_layout.size(self);
let tag_val = tag_size.truncate(discr_val);
let tag = ScalarInt::try_from_uint(tag_val, tag_size).unwrap();
Ok(Some((tag, tag_field)))
interp_ok(Some((tag, tag_field)))
}
abi::Variants::Multiple {
@ -274,7 +275,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
} if untagged_variant == variant_index => {
// The untagged variant is implicitly encoded simply by having a
// value that is outside the niche variants.
Ok(None)
interp_ok(None)
}
abi::Variants::Multiple {
@ -299,7 +300,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let tag = self
.binary_op(mir::BinOp::Add, &variant_index_relative_val, &niche_start_val)?
.to_scalar_int()?;
Ok(Some((tag, tag_field)))
interp_ok(Some((tag, tag_field)))
}
}
}

49
compiler/rustc_const_eval/src/interpret/eval_context.rs
View file

@ -19,9 +19,9 @@ use rustc_trait_selection::traits::ObligationCtxt;
use tracing::{debug, instrument, trace};
use super::{
Frame, FrameInfo, GlobalId, InterpErrorInfo, InterpResult, MPlaceTy, Machine, MemPlaceMeta,
Memory, OpTy, Place, PlaceTy, PointerArithmetic, Projectable, Provenance, err_inval,
throw_inval, throw_ub, throw_ub_custom,
Frame, FrameInfo, GlobalId, InterpError, InterpErrorInfo, InterpResult, MPlaceTy, Machine,
MemPlaceMeta, Memory, OpTy, Place, PlaceTy, PointerArithmetic, Projectable, Provenance,
err_inval, interp_ok, throw_inval, throw_ub, throw_ub_custom,
};
use crate::{ReportErrorExt, fluent_generated as fluent, util};
@ -73,7 +73,7 @@ where
}
impl<'tcx, M: Machine<'tcx>> LayoutOfHelpers<'tcx> for InterpCx<'tcx, M> {
type LayoutOfResult = InterpResult<'tcx, TyAndLayout<'tcx>>;
type LayoutOfResult = Result<TyAndLayout<'tcx>, InterpError<'tcx>>;
#[inline]
fn layout_tcx_at_span(&self) -> Span {
@ -82,29 +82,24 @@ impl<'tcx, M: Machine<'tcx>> LayoutOfHelpers<'tcx> for InterpCx<'tcx, M> {
}
#[inline]
fn handle_layout_err(
&self,
err: LayoutError<'tcx>,
_: Span,
_: Ty<'tcx>,
) -> InterpErrorInfo<'tcx> {
err_inval!(Layout(err)).into()
fn handle_layout_err(&self, err: LayoutError<'tcx>, _: Span, _: Ty<'tcx>) -> InterpError<'tcx> {
err_inval!(Layout(err))
}
}
impl<'tcx, M: Machine<'tcx>> FnAbiOfHelpers<'tcx> for InterpCx<'tcx, M> {
type FnAbiOfResult = InterpResult<'tcx, &'tcx FnAbi<'tcx, Ty<'tcx>>>;
type FnAbiOfResult = Result<&'tcx FnAbi<'tcx, Ty<'tcx>>, InterpError<'tcx>>;
fn handle_fn_abi_err(
&self,
err: FnAbiError<'tcx>,
_span: Span,
_fn_abi_request: FnAbiRequest<'tcx>,
) -> InterpErrorInfo<'tcx> {
) -> InterpError<'tcx> {
match err {
FnAbiError::Layout(err) => err_inval!(Layout(err)).into(),
FnAbiError::Layout(err) => err_inval!(Layout(err)),
FnAbiError::AdjustForForeignAbi(err) => {
err_inval!(FnAbiAdjustForForeignAbi(err)).into()
err_inval!(FnAbiAdjustForForeignAbi(err))
}
}
}
@ -160,7 +155,7 @@ pub(super) fn from_known_layout<'tcx>(
);
}
}
Ok(known_layout)
interp_ok(known_layout)
}
}
}
@ -262,7 +257,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if let Some(err) = body.tainted_by_errors {
throw_inval!(AlreadyReported(ReportedErrorInfo::tainted_by_errors(err)));
}
Ok(body)
interp_ok(body)
}
/// Call this on things you got out of the MIR (so it is as generic as the current
@ -305,7 +300,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
trace!("param_env: {:#?}", self.param_env);
trace!("args: {:#?}", args);
match ty::Instance::try_resolve(*self.tcx, self.param_env, def, args) {
Ok(Some(instance)) => Ok(instance),
Ok(Some(instance)) => interp_ok(instance),
Ok(None) => throw_inval!(TooGeneric),
// FIXME(eddyb) this could be a bit more specific than `AlreadyReported`.
@ -401,7 +396,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
layout: &TyAndLayout<'tcx>,
) -> InterpResult<'tcx, Option<(Size, Align)>> {
if layout.is_sized() {
return Ok(Some((layout.size, layout.align.abi)));
return interp_ok(Some((layout.size, layout.align.abi)));
}
match layout.ty.kind() {
ty::Adt(..) | ty::Tuple(..) => {
@ -425,7 +420,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
else {
// A field with an extern type. We don't know the actual dynamic size
// or the alignment.
return Ok(None);
return interp_ok(None);
};
// # First compute the dynamic alignment
@ -456,12 +451,12 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if full_size > self.max_size_of_val() {
throw_ub!(InvalidMeta(InvalidMetaKind::TooBig));
}
Ok(Some((full_size, full_align)))
interp_ok(Some((full_size, full_align)))
}
ty::Dynamic(expected_trait, _, ty::Dyn) => {
let vtable = metadata.unwrap_meta().to_pointer(self)?;
// Read size and align from vtable (already checks size).
Ok(Some(self.get_vtable_size_and_align(vtable, Some(expected_trait))?))
interp_ok(Some(self.get_vtable_size_and_align(vtable, Some(expected_trait))?))
}
ty::Slice(_) | ty::Str => {
@ -474,10 +469,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if size > self.max_size_of_val() {
throw_ub!(InvalidMeta(InvalidMetaKind::SliceTooBig));
}
Ok(Some((size, elem.align.abi)))
interp_ok(Some((size, elem.align.abi)))
}
ty::Foreign(_) => Ok(None),
ty::Foreign(_) => interp_ok(None),
_ => span_bug!(self.cur_span(), "size_and_align_of::<{}> not supported", layout.ty),
}
@ -503,7 +498,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
pub fn return_to_block(&mut self, target: Option<mir::BasicBlock>) -> InterpResult<'tcx> {
if let Some(target) = target {
self.go_to_block(target);
Ok(())
interp_ok(())
} else {
throw_ub!(Unreachable)
}
@ -530,10 +525,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
M::unwind_terminate(self, reason)?;
// This might have pushed a new stack frame, or it terminated execution.
// Either way, `loc` will not be updated.
return Ok(());
return interp_ok(());
}
};
Ok(())
interp_ok(())
}
/// Call a query that can return `ErrorHandled`. Should be used for statics and other globals.

10
compiler/rustc_const_eval/src/interpret/intern.rs
View file

@ -26,7 +26,9 @@ use rustc_span::def_id::LocalDefId;
use rustc_span::sym;
use tracing::{instrument, trace};
use super::{AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy, err_ub};
use super::{
AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy, err_ub, interp_ok,
};
use crate::const_eval;
use crate::errors::NestedStaticInThreadLocal;
@ -307,7 +309,7 @@ pub fn intern_const_alloc_for_constprop<'tcx, T, M: CompileTimeMachine<'tcx, T>>
) -> InterpResult<'tcx, ()> {
if ecx.tcx.try_get_global_alloc(alloc_id).is_some() {
// The constant is already in global memory. Do nothing.
return Ok(());
return interp_ok(());
}
// Move allocation to `tcx`.
if let Some(_) =
@ -318,7 +320,7 @@ pub fn intern_const_alloc_for_constprop<'tcx, T, M: CompileTimeMachine<'tcx, T>>
// proper recursive interning loop -- or just call `intern_const_alloc_recursive`.
panic!("`intern_const_alloc_for_constprop` called on allocation with nested provenance")
}
Ok(())
interp_ok(())
}
impl<'tcx, M: super::intern::CompileTimeMachine<'tcx, !>> InterpCx<'tcx, M> {
@ -342,6 +344,6 @@ impl<'tcx, M: super::intern::CompileTimeMachine<'tcx, !>> InterpCx<'tcx, M> {
panic!("`intern_with_temp_alloc` with nested allocations");
}
}
Ok(alloc_id)
interp_ok(alloc_id)
}
}

34
compiler/rustc_const_eval/src/interpret/intrinsics.rs
View file

@ -18,7 +18,7 @@ use super::util::ensure_monomorphic_enough;
use super::{
Allocation, CheckInAllocMsg, ConstAllocation, GlobalId, ImmTy, InterpCx, InterpResult,
MPlaceTy, Machine, OpTy, Pointer, PointerArithmetic, Provenance, Scalar, err_inval,
err_ub_custom, err_unsup_format, throw_inval, throw_ub_custom, throw_ub_format,
err_ub_custom, err_unsup_format, interp_ok, throw_inval, throw_ub_custom, throw_ub_format,
};
use crate::fluent_generated as fluent;
@ -39,7 +39,7 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>(
) -> InterpResult<'tcx, ConstValue<'tcx>> {
let tp_ty = args.type_at(0);
let name = tcx.item_name(def_id);
Ok(match name {
interp_ok(match name {
sym::type_name => {
ensure_monomorphic_enough(tcx, tp_ty)?;
let alloc = alloc_type_name(tcx, tp_ty);
@ -329,6 +329,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
fluent::const_eval_offset_from_different_allocations,
name = intrinsic_name,
)
.into()
})?;
// Perform division by size to compute return value.
@ -378,7 +379,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
M::panic_nounwind(self, &msg)?;
// Skip the `return_to_block` at the end (we panicked, we do not return).
return Ok(true);
return interp_ok(true);
}
}
sym::simd_insert => {
@ -396,11 +397,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
for i in 0..dest_len {
let place = self.project_index(&dest, i)?;
let value = if i == index {
elem.clone()
} else {
self.project_index(&input, i)?.into()
};
let value =
if i == index { elem.clone() } else { self.project_index(&input, i)? };
self.copy_op(&value, &place)?;
}
}
@ -441,12 +439,12 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
}
// Unsupported intrinsic: skip the return_to_block below.
_ => return Ok(false),
_ => return interp_ok(false),
}
trace!("{:?}", self.dump_place(&dest.clone().into()));
self.return_to_block(ret)?;
Ok(true)
interp_ok(true)
}
pub(super) fn eval_nondiverging_intrinsic(
@ -460,7 +458,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if !cond {
throw_ub_custom!(fluent::const_eval_assume_false);
}
Ok(())
interp_ok(())
}
NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
count,
@ -502,7 +500,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
}
_ => bug!("not a numeric intrinsic: {}", name),
};
Ok(Scalar::from_uint(bits_out, ret_layout.size))
interp_ok(Scalar::from_uint(bits_out, ret_layout.size))
}
pub fn exact_div(
@ -543,7 +541,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let (val, overflowed) =
self.binary_op(mir_op.wrapping_to_overflowing().unwrap(), l, r)?.to_scalar_pair();
Ok(if overflowed.to_bool()? {
interp_ok(if overflowed.to_bool()? {
let size = l.layout.size;
if l.layout.abi.is_signed() {
// For signed ints the saturated value depends on the sign of the first
@ -585,7 +583,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// The offset must be in bounds starting from `ptr`.
self.check_ptr_access_signed(ptr, offset_bytes, CheckInAllocMsg::PointerArithmeticTest)?;
// This also implies that there is no overflow, so we are done.
Ok(ptr.wrapping_signed_offset(offset_bytes, self))
interp_ok(ptr.wrapping_signed_offset(offset_bytes, self))
}
/// Copy `count*size_of::<T>()` many bytes from `*src` to `*dst`.
@ -631,7 +629,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
self.copy_op(&right, &left)?;
self.copy_op(&temp, &right)?;
self.deallocate_ptr(temp.ptr(), None, kind)?;
Ok(())
interp_ok(())
}
pub fn write_bytes_intrinsic(
@ -672,7 +670,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// `Ordering`'s discriminants are -1/0/+1, so casting does the right thing.
let result = Ord::cmp(left_bytes, right_bytes) as i32;
Ok(Scalar::from_i32(result))
interp_ok(Scalar::from_i32(result))
}
pub(crate) fn raw_eq_intrinsic(
@ -690,13 +688,13 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
this.check_ptr_align(ptr, layout.align.abi)?;
let Some(alloc_ref) = self.get_ptr_alloc(ptr, layout.size)? else {
// zero-sized access
return Ok(&[]);
return interp_ok(&[]);
};
alloc_ref.get_bytes_strip_provenance()
};
let lhs_bytes = get_bytes(self, lhs)?;
let rhs_bytes = get_bytes(self, rhs)?;
Ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
interp_ok(Scalar::from_bool(lhs_bytes == rhs_bytes))
}
}

49
compiler/rustc_const_eval/src/interpret/machine.rs
View file

@ -20,7 +20,8 @@ use rustc_target::spec::abi::Abi as CallAbi;
use super::{
AllocBytes, AllocId, AllocKind, AllocRange, Allocation, CTFE_ALLOC_SALT, ConstAllocation,
CtfeProvenance, FnArg, Frame, ImmTy, InterpCx, InterpResult, MPlaceTy, MemoryKind,
Misalignment, OpTy, PlaceTy, Pointer, Provenance, RangeSet, throw_unsup, throw_unsup_format,
Misalignment, OpTy, PlaceTy, Pointer, Provenance, RangeSet, interp_ok, throw_unsup,
throw_unsup_format,
};
/// Data returned by [`Machine::after_stack_pop`], and consumed by
@ -185,7 +186,7 @@ pub trait Machine<'tcx>: Sized {
ecx: &InterpCx<'tcx, Self>,
instance: ty::InstanceKind<'tcx>,
) -> InterpResult<'tcx, &'tcx mir::Body<'tcx>> {
Ok(ecx.tcx.instance_mir(instance))
interp_ok(ecx.tcx.instance_mir(instance))
}
/// Entry point to all function calls.
@ -280,7 +281,7 @@ pub trait Machine<'tcx>: Sized {
/// Called before a basic block terminator is executed.
#[inline]
fn before_terminator(_ecx: &mut InterpCx<'tcx, Self>) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Determines the result of a `NullaryOp::UbChecks` invocation.
@ -290,7 +291,7 @@ pub trait Machine<'tcx>: Sized {
/// You can use this to detect long or endlessly running programs.
#[inline]
fn increment_const_eval_counter(_ecx: &mut InterpCx<'tcx, Self>) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Called before a global allocation is accessed.
@ -304,7 +305,7 @@ pub trait Machine<'tcx>: Sized {
_static_def_id: Option<DefId>,
_is_write: bool,
) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Return the `AllocId` for the given thread-local static in the current thread.
@ -422,7 +423,7 @@ pub trait Machine<'tcx>: Sized {
_prov: (AllocId, Self::ProvenanceExtra),
_range: AllocRange,
) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Hook for performing extra checks on any memory read access,
@ -433,7 +434,7 @@ pub trait Machine<'tcx>: Sized {
/// Used to prevent statics from self-initializing by reading from their own memory
/// as it is being initialized.
fn before_alloc_read(_ecx: &InterpCx<'tcx, Self>, _alloc_id: AllocId) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Hook for performing extra checks on a memory write access.
@ -446,7 +447,7 @@ pub trait Machine<'tcx>: Sized {
_prov: (AllocId, Self::ProvenanceExtra),
_range: AllocRange,
) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Hook for performing extra operations on a memory deallocation.
@ -460,7 +461,7 @@ pub trait Machine<'tcx>: Sized {
_align: Align,
_kind: MemoryKind<Self::MemoryKind>,
) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Executes a retagging operation for a single pointer.
@ -471,7 +472,7 @@ pub trait Machine<'tcx>: Sized {
_kind: mir::RetagKind,
val: &ImmTy<'tcx, Self::Provenance>,
) -> InterpResult<'tcx, ImmTy<'tcx, Self::Provenance>> {
Ok(val.clone())
interp_ok(val.clone())
}
/// Executes a retagging operation on a compound value.
@ -482,7 +483,7 @@ pub trait Machine<'tcx>: Sized {
_kind: mir::RetagKind,
_place: &PlaceTy<'tcx, Self::Provenance>,
) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Called on places used for in-place function argument and return value handling.
@ -516,7 +517,7 @@ pub trait Machine<'tcx>: Sized {
/// Called immediately after a stack frame got pushed and its locals got initialized.
fn after_stack_push(_ecx: &mut InterpCx<'tcx, Self>) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Called just before the return value is copied to the caller-provided return place.
@ -524,7 +525,7 @@ pub trait Machine<'tcx>: Sized {
_ecx: &InterpCx<'tcx, Self>,
_frame: &Frame<'tcx, Self::Provenance, Self::FrameExtra>,
) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Called immediately after a stack frame got popped, but before jumping back to the caller.
@ -537,14 +538,14 @@ pub trait Machine<'tcx>: Sized {
) -> InterpResult<'tcx, ReturnAction> {
// By default, we do not support unwinding from panics
assert!(!unwinding);
Ok(ReturnAction::Normal)
interp_ok(ReturnAction::Normal)
}
/// Called immediately after an "immediate" local variable is read
/// (i.e., this is called for reads that do not end up accessing addressable memory).
#[inline(always)]
fn after_local_read(_ecx: &InterpCx<'tcx, Self>, _local: mir::Local) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Called immediately after an "immediate" local variable is assigned a new value
@ -556,7 +557,7 @@ pub trait Machine<'tcx>: Sized {
_local: mir::Local,
_storage_live: bool,
) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Called immediately after actual memory was allocated for a local
@ -567,7 +568,7 @@ pub trait Machine<'tcx>: Sized {
_local: mir::Local,
_mplace: &MPlaceTy<'tcx, Self::Provenance>,
) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Evaluate the given constant. The `eval` function will do all the required evaluation,
@ -645,7 +646,7 @@ pub macro compile_time_machine(<$tcx: lifetime>) {
) -> InterpResult<$tcx> {
// For now we don't do any checking here. We can't use `tcx.sess` because that can differ
// between crates, and we need to ensure that const-eval always behaves the same.
Ok(())
interp_ok(())
}
#[inline(always)]
@ -665,7 +666,7 @@ pub macro compile_time_machine(<$tcx: lifetime>) {
fn ub_checks(_ecx: &InterpCx<$tcx, Self>) -> InterpResult<$tcx, bool> {
// We can't look at `tcx.sess` here as that can differ across crates, which can lead to
// unsound differences in evaluating the same constant at different instantiation sites.
Ok(true)
interp_ok(true)
}
#[inline(always)]
@ -675,7 +676,7 @@ pub macro compile_time_machine(<$tcx: lifetime>) {
alloc: &'b Allocation,
) -> InterpResult<$tcx, Cow<'b, Allocation<Self::Provenance>>> {
// Overwrite default implementation: no need to adjust anything.
Ok(Cow::Borrowed(alloc))
interp_ok(Cow::Borrowed(alloc))
}
fn init_alloc_extra(
@ -685,7 +686,7 @@ pub macro compile_time_machine(<$tcx: lifetime>) {
_size: Size,
_align: Align,
) -> InterpResult<$tcx, Self::AllocExtra> {
Ok(())
interp_ok(())
}
fn extern_static_pointer(
@ -693,7 +694,7 @@ pub macro compile_time_machine(<$tcx: lifetime>) {
def_id: DefId,
) -> InterpResult<$tcx, Pointer> {
// Use the `AllocId` associated with the `DefId`. Any actual *access* will fail.
Ok(Pointer::new(ecx.tcx.reserve_and_set_static_alloc(def_id).into(), Size::ZERO))
interp_ok(Pointer::new(ecx.tcx.reserve_and_set_static_alloc(def_id).into(), Size::ZERO))
}
#[inline(always)]
@ -702,7 +703,7 @@ pub macro compile_time_machine(<$tcx: lifetime>) {
ptr: Pointer<CtfeProvenance>,
_kind: Option<MemoryKind<Self::MemoryKind>>,
) -> InterpResult<$tcx, Pointer<CtfeProvenance>> {
Ok(ptr)
interp_ok(ptr)
}
#[inline(always)]
@ -713,7 +714,7 @@ pub macro compile_time_machine(<$tcx: lifetime>) {
// Allow these casts, but make the pointer not dereferenceable.
// (I.e., they behave like transmutation.)
// This is correct because no pointers can ever be exposed in compile-time evaluation.
Ok(Pointer::from_addr_invalid(addr))
interp_ok(Pointer::from_addr_invalid(addr))
}
#[inline(always)]

148
compiler/rustc_const_eval/src/interpret/memory.rs
View file

@ -23,7 +23,7 @@ use tracing::{debug, instrument, trace};
use super::{
AllocBytes, AllocId, AllocMap, AllocRange, Allocation, CheckAlignMsg, CheckInAllocMsg,
CtfeProvenance, GlobalAlloc, InterpCx, InterpResult, Machine, MayLeak, Misalignment, Pointer,
PointerArithmetic, Provenance, Scalar, alloc_range, err_ub, err_ub_custom, throw_ub,
PointerArithmetic, Provenance, Scalar, alloc_range, err_ub, err_ub_custom, interp_ok, throw_ub,
throw_ub_custom, throw_unsup, throw_unsup_format,
};
use crate::fluent_generated as fluent;
@ -82,7 +82,7 @@ pub enum FnVal<'tcx, Other> {
impl<'tcx, Other> FnVal<'tcx, Other> {
pub fn as_instance(self) -> InterpResult<'tcx, Instance<'tcx>> {
match self {
FnVal::Instance(instance) => Ok(instance),
FnVal::Instance(instance) => interp_ok(instance),
FnVal::Other(_) => {
throw_unsup_format!("'foreign' function pointers are not supported in this context")
}
@ -284,7 +284,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
self.mem_copy(ptr, new_ptr.into(), old_size.min(new_size), /*nonoverlapping*/ true)?;
self.deallocate_ptr(ptr, old_size_and_align, kind)?;
Ok(new_ptr)
interp_ok(new_ptr)
}
#[instrument(skip(self), level = "debug")]
@ -330,8 +330,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
)
}
None => err_ub!(PointerUseAfterFree(alloc_id, CheckInAllocMsg::MemoryAccessTest)),
}
.into());
})
.into();
};
if alloc.mutability.is_not() {
@ -376,7 +376,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
bug!("Nothing can be deallocated twice");
}
Ok(())
interp_ok(())
}
/// Internal helper function to determine the allocation and offset of a pointer (if any).
@ -395,7 +395,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
|this, alloc_id, offset, prov| {
let (size, align) = this
.get_live_alloc_size_and_align(alloc_id, CheckInAllocMsg::MemoryAccessTest)?;
Ok((size, align, (alloc_id, offset, prov)))
interp_ok((size, align, (alloc_id, offset, prov)))
},
)
}
@ -412,9 +412,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let size = i64::try_from(size.bytes()).unwrap(); // it would be an error to even ask for more than isize::MAX bytes
Self::check_and_deref_ptr(self, ptr, size, msg, |this, alloc_id, _, _| {
let (size, align) = this.get_live_alloc_size_and_align(alloc_id, msg)?;
Ok((size, align, ()))
interp_ok((size, align, ()))
})?;
Ok(())
interp_ok(())
}
/// Check whether the given pointer points to live memory for a signed amount of bytes.
@ -428,9 +428,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
) -> InterpResult<'tcx> {
Self::check_and_deref_ptr(self, ptr, size, msg, |this, alloc_id, _, _| {
let (size, align) = this.get_live_alloc_size_and_align(alloc_id, msg)?;
Ok((size, align, ()))
interp_ok((size, align, ()))
})?;
Ok(())
interp_ok(())
}
/// Low-level helper function to check if a ptr is in-bounds and potentially return a reference
@ -455,10 +455,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
) -> InterpResult<'tcx, Option<T>> {
// Everything is okay with size 0.
if size == 0 {
return Ok(None);
return interp_ok(None);
}
Ok(match this.borrow().ptr_try_get_alloc_id(ptr, size) {
interp_ok(match this.borrow().ptr_try_get_alloc_id(ptr, size) {
Err(addr) => {
// We couldn't get a proper allocation.
throw_ub!(DanglingIntPointer { addr, inbounds_size: size, msg });
@ -498,7 +498,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if let Some(misaligned) = misaligned {
throw_ub!(AlignmentCheckFailed(misaligned, msg))
}
Ok(())
interp_ok(())
}
pub(super) fn is_ptr_misaligned(
@ -634,7 +634,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// `get_global_alloc` that we can actually use directly without inserting anything anywhere.
// So the error type is `InterpResult<'tcx, &Allocation<M::Provenance>>`.
let a = self.memory.alloc_map.get_or(id, || {
let alloc = self.get_global_alloc(id, /*is_write*/ false).map_err(Err)?;
// We have to funnel the `InterpErrorInfo` through a `Result` to match the `get_or` API,
// so we use `report_err` for that.
let alloc = self.get_global_alloc(id, /*is_write*/ false).report_err().map_err(Err)?;
match alloc {
Cow::Borrowed(alloc) => {
// We got a ref, cheaply return that as an "error" so that the
@ -653,8 +655,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
});
// Now unpack that funny error type
match a {
Ok(a) => Ok(&a.1),
Err(a) => a,
Ok(a) => interp_ok(&a.1),
Err(a) => a.into(),
}
}
@ -662,7 +664,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
/// The caller is responsible for calling the access hooks!
pub fn get_alloc_bytes_unchecked_raw(&self, id: AllocId) -> InterpResult<'tcx, *const u8> {
let alloc = self.get_alloc_raw(id)?;
Ok(alloc.get_bytes_unchecked_raw())
interp_ok(alloc.get_bytes_unchecked_raw())
}
/// Bounds-checked *but not align-checked* allocation access.
@ -680,7 +682,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
CheckInAllocMsg::MemoryAccessTest,
|this, alloc_id, offset, prov| {
let alloc = this.get_alloc_raw(alloc_id)?;
Ok((alloc.size(), alloc.align, (alloc_id, offset, prov, alloc)))
interp_ok((alloc.size(), alloc.align, (alloc_id, offset, prov, alloc)))
},
)?;
// We want to call the hook on *all* accesses that involve an AllocId, including zero-sized
@ -703,20 +705,20 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
range,
)?;
}
Ok(Some(AllocRef { alloc, range, tcx: *self.tcx, alloc_id }))
interp_ok(Some(AllocRef { alloc, range, tcx: *self.tcx, alloc_id }))
} else {
Ok(None)
interp_ok(None)
}
}
/// Return the `extra` field of the given allocation.
pub fn get_alloc_extra<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, &'a M::AllocExtra> {
Ok(&self.get_alloc_raw(id)?.extra)
interp_ok(&self.get_alloc_raw(id)?.extra)
}
/// Return the `mutability` field of the given allocation.
pub fn get_alloc_mutability<'a>(&'a self, id: AllocId) -> InterpResult<'tcx, Mutability> {
Ok(self.get_alloc_raw(id)?.mutability)
interp_ok(self.get_alloc_raw(id)?.mutability)
}
/// Gives raw mutable access to the `Allocation`, without bounds or alignment checks.
@ -750,7 +752,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if alloc.mutability.is_not() {
throw_ub!(WriteToReadOnly(id))
}
Ok((alloc, &mut self.machine))
interp_ok((alloc, &mut self.machine))
}
/// Gives raw, mutable access to the `Allocation` address, without bounds or alignment checks.
@ -760,7 +762,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
id: AllocId,
) -> InterpResult<'tcx, *mut u8> {
let alloc = self.get_alloc_raw_mut(id)?.0;
Ok(alloc.get_bytes_unchecked_raw_mut())
interp_ok(alloc.get_bytes_unchecked_raw_mut())
}
/// Bounds-checked *but not align-checked* allocation access.
@ -781,7 +783,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
CheckInAllocMsg::MemoryAccessTest,
|this, alloc_id, offset, prov| {
let (alloc, machine) = this.get_alloc_raw_mut(alloc_id)?;
Ok((alloc.size(), alloc.align, (alloc_id, offset, prov, alloc, machine)))
interp_ok((alloc.size(), alloc.align, (alloc_id, offset, prov, alloc, machine)))
},
)?;
@ -790,9 +792,9 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if !validation_in_progress {
M::before_memory_write(tcx, machine, &mut alloc.extra, (alloc_id, prov), range)?;
}
Ok(Some(AllocRefMut { alloc, range, tcx: *tcx, alloc_id }))
interp_ok(Some(AllocRefMut { alloc, range, tcx: *tcx, alloc_id }))
} else {
Ok(None)
interp_ok(None)
}
}
@ -802,7 +804,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
id: AllocId,
) -> InterpResult<'tcx, (&'a mut M::AllocExtra, &'a mut M)> {
let (alloc, machine) = self.get_alloc_raw_mut(id)?;
Ok((&mut alloc.extra, machine))
interp_ok((&mut alloc.extra, machine))
}
/// Check whether an allocation is live. This is faster than calling
@ -904,7 +906,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if matches!(kind, AllocKind::Dead) {
throw_ub!(PointerUseAfterFree(id, msg))
}
Ok((size, align))
interp_ok((size, align))
}
fn get_fn_alloc(&self, id: AllocId) -> Option<FnVal<'tcx, M::ExtraFnVal>> {
@ -928,7 +930,8 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
throw_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset)))
}
self.get_fn_alloc(alloc_id)
.ok_or_else(|| err_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset))).into())
.ok_or_else(|| err_ub!(InvalidFunctionPointer(Pointer::new(alloc_id, offset))))
.into()
}
/// Get the dynamic type of the given vtable pointer.
@ -951,12 +954,12 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if let Some(expected_dyn_type) = expected_trait {
self.check_vtable_for_type(vtable_dyn_type, expected_dyn_type)?;
}
Ok(ty)
interp_ok(ty)
}
pub fn alloc_mark_immutable(&mut self, id: AllocId) -> InterpResult<'tcx> {
self.get_alloc_raw_mut(id)?.0.mutability = Mutability::Not;
Ok(())
interp_ok(())
}
/// Create a lazy debug printer that prints the given allocation and all allocations it points
@ -1144,10 +1147,11 @@ impl<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes>
pub fn write_scalar(&mut self, range: AllocRange, val: Scalar<Prov>) -> InterpResult<'tcx> {
let range = self.range.subrange(range);
debug!("write_scalar at {:?}{range:?}: {val:?}", self.alloc_id);
Ok(self
.alloc
self.alloc
.write_scalar(&self.tcx, range, val)
.map_err(|e| e.to_interp_error(self.alloc_id))?)
.map_err(|e| e.to_interp_error(self.alloc_id))
.into()
}
/// `offset` is relative to this allocation reference, not the base of the allocation.
@ -1158,26 +1162,27 @@ impl<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes>
/// Mark the given sub-range (relative to this allocation reference) as uninitialized.
pub fn write_uninit(&mut self, range: AllocRange) -> InterpResult<'tcx> {
let range = self.range.subrange(range);
Ok(self
.alloc
self.alloc
.write_uninit(&self.tcx, range)
.map_err(|e| e.to_interp_error(self.alloc_id))?)
.map_err(|e| e.to_interp_error(self.alloc_id))
.into()
}
/// Mark the entire referenced range as uninitialized
pub fn write_uninit_full(&mut self) -> InterpResult<'tcx> {
Ok(self
.alloc
self.alloc
.write_uninit(&self.tcx, self.range)
.map_err(|e| e.to_interp_error(self.alloc_id))?)
.map_err(|e| e.to_interp_error(self.alloc_id))
.into()
}
/// Remove all provenance in the reference range.
pub fn clear_provenance(&mut self) -> InterpResult<'tcx> {
Ok(self
.alloc
self.alloc
.clear_provenance(&self.tcx, self.range)
.map_err(|e| e.to_interp_error(self.alloc_id))?)
.map_err(|e| e.to_interp_error(self.alloc_id))
.into()
}
}
@ -1189,12 +1194,10 @@ impl<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes> AllocRef<'a, 'tcx, Pr
read_provenance: bool,
) -> InterpResult<'tcx, Scalar<Prov>> {
let range = self.range.subrange(range);
let res = self
.alloc
self.alloc
.read_scalar(&self.tcx, range, read_provenance)
.map_err(|e| e.to_interp_error(self.alloc_id))?;
debug!("read_scalar at {:?}{range:?}: {res:?}", self.alloc_id);
Ok(res)
.map_err(|e| e.to_interp_error(self.alloc_id))
.into()
}
/// `range` is relative to this allocation reference, not the base of the allocation.
@ -1212,10 +1215,10 @@ impl<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes> AllocRef<'a, 'tcx, Pr
/// `range` is relative to this allocation reference, not the base of the allocation.
pub fn get_bytes_strip_provenance<'b>(&'b self) -> InterpResult<'tcx, &'a [u8]> {
Ok(self
.alloc
self.alloc
.get_bytes_strip_provenance(&self.tcx, self.range)
.map_err(|e| e.to_interp_error(self.alloc_id))?)
.map_err(|e| e.to_interp_error(self.alloc_id))
.into()
}
/// Returns whether the allocation has provenance anywhere in the range of the `AllocRef`.
@ -1236,14 +1239,16 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
) -> InterpResult<'tcx, &[u8]> {
let Some(alloc_ref) = self.get_ptr_alloc(ptr, size)? else {
// zero-sized access
return Ok(&[]);
return interp_ok(&[]);
};
// Side-step AllocRef and directly access the underlying bytes more efficiently.
// (We are staying inside the bounds here so all is good.)
Ok(alloc_ref
.alloc
.get_bytes_strip_provenance(&alloc_ref.tcx, alloc_ref.range)
.map_err(|e| e.to_interp_error(alloc_ref.alloc_id))?)
interp_ok(
alloc_ref
.alloc
.get_bytes_strip_provenance(&alloc_ref.tcx, alloc_ref.range)
.map_err(|e| e.to_interp_error(alloc_ref.alloc_id))?,
)
}
/// Writes the given stream of bytes into memory.
@ -1263,7 +1268,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let Some(alloc_ref) = self.get_ptr_alloc_mut(ptr, size)? else {
// zero-sized access
assert_matches!(src.next(), None, "iterator said it was empty but returned an element");
return Ok(());
return interp_ok(());
};
// Side-step AllocRef and directly access the underlying bytes more efficiently.
@ -1279,7 +1284,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
*dest = src.next().expect("iterator was shorter than it said it would be");
}
assert_matches!(src.next(), None, "iterator was longer than it said it would be");
Ok(())
interp_ok(())
}
pub fn mem_copy(
@ -1316,7 +1321,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// Source alloc preparations and access hooks.
let Some((src_alloc_id, src_offset, src_prov)) = src_parts else {
// Zero-sized *source*, that means dest is also zero-sized and we have nothing to do.
return Ok(());
return interp_ok(());
};
let src_alloc = self.get_alloc_raw(src_alloc_id)?;
let src_range = alloc_range(src_offset, size);
@ -1332,7 +1337,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// We already did the source checks and called the hooks so we are good to return early.
let Some((dest_alloc_id, dest_offset, dest_prov)) = dest_parts else {
// Zero-sized *destination*.
return Ok(());
return interp_ok(());
};
// Prepare getting source provenance.
@ -1375,7 +1380,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
.write_uninit(&tcx, dest_range)
.map_err(|e| e.to_interp_error(dest_alloc_id))?;
// We can forget about the provenance, this is all not initialized anyway.
return Ok(());
return interp_ok(());
}
// SAFE: The above indexing would have panicked if there weren't at least `size` bytes
@ -1432,7 +1437,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// copy the provenance to the destination
dest_alloc.provenance_apply_copy(provenance);
Ok(())
interp_ok(())
}
}
@ -1441,7 +1446,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
/// Test if this value might be null.
/// If the machine does not support ptr-to-int casts, this is conservative.
pub fn scalar_may_be_null(&self, scalar: Scalar<M::Provenance>) -> InterpResult<'tcx, bool> {
Ok(match scalar.try_to_scalar_int() {
interp_ok(match scalar.try_to_scalar_int() {
Ok(int) => int.is_null(),
Err(_) => {
// Can only happen during CTFE.
@ -1508,13 +1513,14 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
ptr: Pointer<Option<M::Provenance>>,
size: i64,
) -> InterpResult<'tcx, (AllocId, Size, M::ProvenanceExtra)> {
self.ptr_try_get_alloc_id(ptr, size).map_err(|offset| {
err_ub!(DanglingIntPointer {
addr: offset,
inbounds_size: size,
msg: CheckInAllocMsg::InboundsTest
self.ptr_try_get_alloc_id(ptr, size)
.map_err(|offset| {
err_ub!(DanglingIntPointer {
addr: offset,
inbounds_size: size,
msg: CheckInAllocMsg::InboundsTest
})
})
.into()
})
}
}

52
compiler/rustc_const_eval/src/interpret/operand.rs
View file

@ -16,7 +16,7 @@ use tracing::trace;
use super::{
CtfeProvenance, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace, MemPlaceMeta, OffsetMode,
PlaceTy, Pointer, Projectable, Provenance, Scalar, alloc_range, err_ub, from_known_layout,
mir_assign_valid_types, throw_ub,
interp_ok, mir_assign_valid_types, throw_ub,
};
/// An `Immediate` represents a single immediate self-contained Rust value.
@ -118,6 +118,7 @@ impl<Prov: Provenance> Immediate<Prov> {
(Immediate::Scalar(scalar), Abi::Scalar(s)) => {
assert_eq!(scalar.size(), s.size(cx));
if !matches!(s.primitive(), abi::Pointer(..)) {
// This is not a pointer, it should not carry provenance.
assert!(matches!(scalar, Scalar::Int(..)));
}
}
@ -149,7 +150,7 @@ impl<Prov: Provenance> Immediate<Prov> {
}
Immediate::Uninit => {}
}
Ok(())
interp_ok(())
}
}
@ -307,7 +308,7 @@ impl<'tcx, Prov: Provenance> ImmTy<'tcx, Prov> {
data_size: s.size().bytes(),
}));
}
Ok(s)
interp_ok(s)
}
#[inline]
@ -430,7 +431,7 @@ impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for ImmTy<'tcx, Prov> {
ecx: &InterpCx<'tcx, M>,
) -> InterpResult<'tcx, Self> {
assert_matches!(meta, MemPlaceMeta::None); // we can't store this anywhere anyway
Ok(self.offset_(offset, layout, ecx))
interp_ok(self.offset_(offset, layout, ecx))
}
#[inline(always)]
@ -438,7 +439,7 @@ impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for ImmTy<'tcx, Prov> {
&self,
_ecx: &InterpCx<'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
Ok(self.clone().into())
interp_ok(self.clone().into())
}
}
@ -514,11 +515,13 @@ impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for OpTy<'tcx, Prov> {
ecx: &InterpCx<'tcx, M>,
) -> InterpResult<'tcx, Self> {
match self.as_mplace_or_imm() {
Left(mplace) => Ok(mplace.offset_with_meta(offset, mode, meta, layout, ecx)?.into()),
Left(mplace) => {
interp_ok(mplace.offset_with_meta(offset, mode, meta, layout, ecx)?.into())
}
Right(imm) => {
assert_matches!(meta, MemPlaceMeta::None); // no place to store metadata here
// Every part of an uninit is uninit.
Ok(imm.offset_(offset, layout, ecx).into())
interp_ok(imm.offset_(offset, layout, ecx).into())
}
}
}
@ -528,7 +531,7 @@ impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for OpTy<'tcx, Prov> {
&self,
_ecx: &InterpCx<'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
Ok(self.clone())
interp_ok(self.clone())
}
}
@ -543,12 +546,12 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
) -> InterpResult<'tcx, Option<ImmTy<'tcx, M::Provenance>>> {
if mplace.layout.is_unsized() {
// Don't touch unsized
return Ok(None);
return interp_ok(None);
}
let Some(alloc) = self.get_place_alloc(mplace)? else {
// zero-sized type can be left uninit
return Ok(Some(ImmTy::uninit(mplace.layout)));
return interp_ok(Some(ImmTy::uninit(mplace.layout)));
};
// It may seem like all types with `Scalar` or `ScalarPair` ABI are fair game at this point.
@ -557,7 +560,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// case where some of the bytes are initialized and others are not. So, we need an extra
// check that walks over the type of `mplace` to make sure it is truly correct to treat this
// like a `Scalar` (or `ScalarPair`).
Ok(match mplace.layout.abi {
interp_ok(match mplace.layout.abi {
Abi::Scalar(abi::Scalar::Initialized { value: s, .. }) => {
let size = s.size(self);
assert_eq!(size, mplace.layout.size, "abi::Scalar size does not match layout size");
@ -606,7 +609,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
&self,
src: &impl Projectable<'tcx, M::Provenance>,
) -> InterpResult<'tcx, Either<MPlaceTy<'tcx, M::Provenance>, ImmTy<'tcx, M::Provenance>>> {
Ok(match src.to_op(self)?.as_mplace_or_imm() {
interp_ok(match src.to_op(self)?.as_mplace_or_imm() {
Left(ref mplace) => {
if let Some(val) = self.read_immediate_from_mplace_raw(mplace)? {
Right(val)
@ -637,7 +640,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
if matches!(*imm, Immediate::Uninit) {
throw_ub!(InvalidUninitBytes(None));
}
Ok(imm)
interp_ok(imm)
}
/// Read a scalar from a place
@ -645,7 +648,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
&self,
op: &impl Projectable<'tcx, M::Provenance>,
) -> InterpResult<'tcx, Scalar<M::Provenance>> {
Ok(self.read_immediate(op)?.to_scalar())
interp_ok(self.read_immediate(op)?.to_scalar())
}
// Pointer-sized reads are fairly common and need target layout access, so we wrap them in
@ -678,7 +681,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let len = mplace.len(self)?;
let bytes = self.read_bytes_ptr_strip_provenance(mplace.ptr(), Size::from_bytes(len))?;
let str = std::str::from_utf8(bytes).map_err(|err| err_ub!(InvalidStr(err)))?;
Ok(str)
interp_ok(str)
}
/// Read from a local of the current frame.
@ -698,7 +701,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
assert!(!layout.is_unsized());
}
M::after_local_read(self, local)?;
Ok(OpTy { op, layout })
interp_ok(OpTy { op, layout })
}
/// Every place can be read from, so we can turn them into an operand.
@ -709,12 +712,12 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
place: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
match place.as_mplace_or_local() {
Left(mplace) => Ok(mplace.into()),
Left(mplace) => interp_ok(mplace.into()),
Right((local, offset, locals_addr, _)) => {
debug_assert!(place.layout.is_sized()); // only sized locals can ever be `Place::Local`.
debug_assert_eq!(locals_addr, self.frame().locals_addr());
let base = self.local_to_op(local, None)?;
Ok(match offset {
interp_ok(match offset {
Some(offset) => base.offset(offset, place.layout, self)?,
None => {
// In the common case this hasn't been projected.
@ -764,7 +767,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
)
}
}
Ok(op)
interp_ok(op)
}
/// Evaluate the operand, returning a place where you can then find the data.
@ -794,7 +797,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
}
};
trace!("{:?}: {:?}", mir_op, op);
Ok(op)
interp_ok(op)
}
pub(crate) fn const_val_to_op(
@ -805,12 +808,13 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
// Other cases need layout.
let adjust_scalar = |scalar| -> InterpResult<'tcx, _> {
Ok(match scalar {
interp_ok(match scalar {
Scalar::Ptr(ptr, size) => Scalar::Ptr(self.global_root_pointer(ptr)?, size),
Scalar::Int(int) => Scalar::Int(int),
})
};
let layout = from_known_layout(self.tcx, self.param_env, layout, || self.layout_of(ty))?;
let layout =
from_known_layout(self.tcx, self.param_env, layout, || self.layout_of(ty).into())?;
let imm = match val_val {
mir::ConstValue::Indirect { alloc_id, offset } => {
// This is const data, no mutation allowed.
@ -818,7 +822,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
CtfeProvenance::from(alloc_id).as_immutable(),
offset,
))?;
return Ok(self.ptr_to_mplace(ptr.into(), layout).into());
return interp_ok(self.ptr_to_mplace(ptr.into(), layout).into());
}
mir::ConstValue::Scalar(x) => adjust_scalar(x)?.into(),
mir::ConstValue::ZeroSized => Immediate::Uninit,
@ -829,7 +833,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
Immediate::new_slice(self.global_root_pointer(ptr)?.into(), meta, self)
}
};
Ok(OpTy { op: Operand::Immediate(imm), layout })
interp_ok(OpTy { op: Operand::Immediate(imm), layout })
}
}

37
compiler/rustc_const_eval/src/interpret/operator.rs
View file

@ -9,7 +9,7 @@ use rustc_span::symbol::sym;
use rustc_target::abi::Size;
use tracing::trace;
use super::{ImmTy, InterpCx, Machine, MemPlaceMeta, throw_ub};
use super::{ImmTy, InterpCx, Machine, MemPlaceMeta, interp_ok, throw_ub};
impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
fn three_way_compare<T: Ord>(&self, lhs: T, rhs: T) -> ImmTy<'tcx, M::Provenance> {
@ -156,7 +156,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
});
}
return Ok(ImmTy::from_scalar_int(result, left.layout));
return interp_ok(ImmTy::from_scalar_int(result, left.layout));
}
// For the remaining ops, the types must be the same on both sides
@ -181,10 +181,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
_ => None,
};
if let Some(op) = op {
return Ok(ImmTy::from_bool(op(&l_signed(), &r_signed()), *self.tcx));
return interp_ok(ImmTy::from_bool(op(&l_signed(), &r_signed()), *self.tcx));
}
if bin_op == Cmp {
return Ok(self.three_way_compare(l_signed(), r_signed()));
return interp_ok(self.three_way_compare(l_signed(), r_signed()));
}
let op: Option<fn(i128, i128) -> (i128, bool)> = match bin_op {
Div if r.is_null() => throw_ub!(DivisionByZero),
@ -221,7 +221,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
throw_ub!(ArithOverflow { intrinsic });
}
let res = ImmTy::from_scalar_int(result, left.layout);
return Ok(if with_overflow {
return interp_ok(if with_overflow {
let overflow = ImmTy::from_bool(overflow, *self.tcx);
ImmTy::from_pair(res, overflow, *self.tcx)
} else {
@ -234,10 +234,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let r = r_unsigned();
if bin_op == Cmp {
return Ok(self.three_way_compare(l, r));
return interp_ok(self.three_way_compare(l, r));
}
Ok(match bin_op {
interp_ok(match bin_op {
Eq => ImmTy::from_bool(l == r, *self.tcx),
Ne => ImmTy::from_bool(l != r, *self.tcx),
@ -339,7 +339,10 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
throw_ub!(PointerArithOverflow)
}
let offset_ptr = self.ptr_offset_inbounds(ptr, offset_bytes)?;
Ok(ImmTy::from_scalar(Scalar::from_maybe_pointer(offset_ptr, self), left.layout))
interp_ok(ImmTy::from_scalar(
Scalar::from_maybe_pointer(offset_ptr, self),
left.layout,
))
}
// Fall back to machine hook so Miri can support more pointer ops.
@ -366,20 +369,20 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
assert_eq!(left.layout.ty, right.layout.ty);
let left = left.to_scalar();
let right = right.to_scalar();
Ok(self.binary_char_op(bin_op, left.to_char()?, right.to_char()?))
interp_ok(self.binary_char_op(bin_op, left.to_char()?, right.to_char()?))
}
ty::Bool => {
assert_eq!(left.layout.ty, right.layout.ty);
let left = left.to_scalar();
let right = right.to_scalar();
Ok(self.binary_bool_op(bin_op, left.to_bool()?, right.to_bool()?))
interp_ok(self.binary_bool_op(bin_op, left.to_bool()?, right.to_bool()?))
}
ty::Float(fty) => {
assert_eq!(left.layout.ty, right.layout.ty);
let layout = left.layout;
let left = left.to_scalar();
let right = right.to_scalar();
Ok(match fty {
interp_ok(match fty {
FloatTy::F16 => {
self.binary_float_op(bin_op, layout, left.to_f16()?, right.to_f16()?)
}
@ -447,7 +450,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
Not => !val,
_ => span_bug!(self.cur_span(), "Invalid bool op {:?}", un_op),
};
Ok(ImmTy::from_bool(res, *self.tcx))
interp_ok(ImmTy::from_bool(res, *self.tcx))
}
ty::Float(fty) => {
let val = val.to_scalar();
@ -462,7 +465,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
FloatTy::F64 => Scalar::from_f64(-val.to_f64()?),
FloatTy::F128 => Scalar::from_f128(-val.to_f128()?),
};
Ok(ImmTy::from_scalar(res, layout))
interp_ok(ImmTy::from_scalar(res, layout))
}
ty::Int(..) => {
let val = val.to_scalar().to_int(layout.size)?;
@ -472,7 +475,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
_ => span_bug!(self.cur_span(), "Invalid integer op {:?}", un_op),
};
let res = ScalarInt::truncate_from_int(res, layout.size).0;
Ok(ImmTy::from_scalar(res.into(), layout))
interp_ok(ImmTy::from_scalar(res.into(), layout))
}
ty::Uint(..) => {
let val = val.to_scalar().to_uint(layout.size)?;
@ -481,12 +484,12 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
_ => span_bug!(self.cur_span(), "Invalid unsigned integer op {:?}", un_op),
};
let res = ScalarInt::truncate_from_uint(res, layout.size).0;
Ok(ImmTy::from_scalar(res.into(), layout))
interp_ok(ImmTy::from_scalar(res.into(), layout))
}
ty::RawPtr(..) | ty::Ref(..) => {
assert_eq!(un_op, PtrMetadata);
let (_, meta) = val.to_scalar_and_meta();
Ok(match meta {
interp_ok(match meta {
MemPlaceMeta::Meta(scalar) => {
let ty = un_op.ty(*self.tcx, val.layout.ty);
let layout = self.layout_of(ty)?;
@ -514,7 +517,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let layout = self.layout_of(arg_ty)?;
let usize_layout = || self.layout_of(self.tcx.types.usize).unwrap();
Ok(match null_op {
interp_ok(match null_op {
SizeOf => {
if !layout.abi.is_sized() {
span_bug!(self.cur_span(), "unsized type for `NullaryOp::SizeOf`");

73
compiler/rustc_const_eval/src/interpret/place.rs
View file

@ -15,7 +15,7 @@ use tracing::{instrument, trace};
use super::{
AllocRef, AllocRefMut, CheckAlignMsg, CtfeProvenance, ImmTy, Immediate, InterpCx, InterpResult,
Machine, MemoryKind, Misalignment, OffsetMode, OpTy, Operand, Pointer, Projectable, Provenance,
Scalar, alloc_range, mir_assign_valid_types,
Scalar, alloc_range, interp_ok, mir_assign_valid_types,
};
#[derive(Copy, Clone, Hash, PartialEq, Eq, Debug)]
@ -90,7 +90,7 @@ impl<Prov: Provenance> MemPlace<Prov> {
}
OffsetMode::Wrapping => self.ptr.wrapping_offset(offset, ecx),
};
Ok(MemPlace { ptr, meta, misaligned: self.misaligned })
interp_ok(MemPlace { ptr, meta, misaligned: self.misaligned })
}
}
@ -163,7 +163,10 @@ impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
layout: TyAndLayout<'tcx>,
ecx: &InterpCx<'tcx, M>,
) -> InterpResult<'tcx, Self> {
Ok(MPlaceTy { mplace: self.mplace.offset_with_meta_(offset, mode, meta, ecx)?, layout })
interp_ok(MPlaceTy {
mplace: self.mplace.offset_with_meta_(offset, mode, meta, ecx)?,
layout,
})
}
#[inline(always)]
@ -171,7 +174,7 @@ impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
&self,
_ecx: &InterpCx<'tcx, M>,
) -> InterpResult<'tcx, OpTy<'tcx, M::Provenance>> {
Ok(self.clone().into())
interp_ok(self.clone().into())
}
}
@ -279,7 +282,7 @@ impl<'tcx, Prov: Provenance> Projectable<'tcx, Prov> for PlaceTy<'tcx, Prov> {
layout: TyAndLayout<'tcx>,
ecx: &InterpCx<'tcx, M>,
) -> InterpResult<'tcx, Self> {
Ok(match self.as_mplace_or_local() {
interp_ok(match self.as_mplace_or_local() {
Left(mplace) => mplace.offset_with_meta(offset, mode, meta, layout, ecx)?.into(),
Right((local, old_offset, locals_addr, _)) => {
debug_assert!(layout.is_sized(), "unsized locals should live in memory");
@ -367,7 +370,7 @@ impl<'tcx, Prov: Provenance> Writeable<'tcx, Prov> for MPlaceTy<'tcx, Prov> {
&self,
_ecx: &mut InterpCx<'tcx, M>,
) -> InterpResult<'tcx, MPlaceTy<'tcx, Prov>> {
Ok(self.clone())
interp_ok(self.clone())
}
}
@ -425,7 +428,7 @@ where
// `ref_to_mplace` is called on raw pointers even if they don't actually get dereferenced;
// we hence can't call `size_and_align_of` since that asserts more validity than we want.
let ptr = ptr.to_pointer(self)?;
Ok(self.ptr_with_meta_to_mplace(ptr, meta, layout, /*unaligned*/ false))
interp_ok(self.ptr_with_meta_to_mplace(ptr, meta, layout, /*unaligned*/ false))
}
/// Turn a mplace into a (thin or wide) mutable raw pointer, pointing to the same space.
@ -437,7 +440,7 @@ where
) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
let imm = mplace.mplace.to_ref(self);
let layout = self.layout_of(Ty::new_mut_ptr(self.tcx.tcx, mplace.layout.ty))?;
Ok(ImmTy::from_immediate(imm, layout))
interp_ok(ImmTy::from_immediate(imm, layout))
}
/// Take an operand, representing a pointer, and dereference it to a place.
@ -458,7 +461,7 @@ where
trace!("deref to {} on {:?}", val.layout.ty, *val);
let mplace = self.ref_to_mplace(&val)?;
Ok(mplace)
interp_ok(mplace)
}
#[inline]
@ -474,7 +477,7 @@ where
// If an access is both OOB and misaligned, we want to see the bounds error.
let a = self.get_ptr_alloc(mplace.ptr(), size)?;
self.check_misalign(mplace.mplace.misaligned, CheckAlignMsg::BasedOn)?;
Ok(a)
interp_ok(a)
}
#[inline]
@ -489,10 +492,10 @@ where
// We check alignment separately, and raise that error *after* checking everything else.
// If an access is both OOB and misaligned, we want to see the bounds error.
// However we have to call `check_misalign` first to make the borrow checker happy.
let misalign_err = self.check_misalign(mplace.mplace.misaligned, CheckAlignMsg::BasedOn);
let a = self.get_ptr_alloc_mut(mplace.ptr(), size)?;
misalign_err?;
Ok(a)
let misalign_res = self.check_misalign(mplace.mplace.misaligned, CheckAlignMsg::BasedOn);
// An error from get_ptr_alloc_mut takes precedence.
let (a, ()) = self.get_ptr_alloc_mut(mplace.ptr(), size).and(misalign_res)?;
interp_ok(a)
}
/// Turn a local in the current frame into a place.
@ -512,7 +515,7 @@ where
Operand::Indirect(mplace) => Place::Ptr(*mplace),
}
};
Ok(PlaceTy { place, layout })
interp_ok(PlaceTy { place, layout })
}
/// Computes a place. You should only use this if you intend to write into this
@ -549,7 +552,7 @@ where
)
}
}
Ok(place)
interp_ok(place)
}
/// Given a place, returns either the underlying mplace or a reference to where the value of
@ -565,7 +568,7 @@ where
(&mut Immediate<M::Provenance>, TyAndLayout<'tcx>, mir::Local),
>,
> {
Ok(match place.to_place().as_mplace_or_local() {
interp_ok(match place.to_place().as_mplace_or_local() {
Left(mplace) => Left(mplace),
Right((local, offset, locals_addr, layout)) => {
if offset.is_some() {
@ -610,7 +613,7 @@ where
)?;
}
Ok(())
interp_ok(())
}
/// Write a scalar to a place
@ -652,6 +655,8 @@ where
M::after_local_write(self, local, /*storage_live*/ false)?;
}
// Double-check that the value we are storing and the local fit to each other.
// Things can ge wrong in quite weird ways when this is violated.
// Unfortunately this is too expensive to do in release builds.
if cfg!(debug_assertions) {
src.assert_matches_abi(local_layout.abi, self);
}
@ -660,7 +665,7 @@ where
self.write_immediate_to_mplace_no_validate(src, mplace.layout, mplace.mplace)?;
}
}
Ok(())
interp_ok(())
}
/// Write an immediate to memory.
@ -672,9 +677,9 @@ where
layout: TyAndLayout<'tcx>,
dest: MemPlace<M::Provenance>,
) -> InterpResult<'tcx> {
if cfg!(debug_assertions) {
value.assert_matches_abi(layout.abi, self);
}
// We use the sizes from `value` below.
// Ensure that matches the type of the place it is written to.
value.assert_matches_abi(layout.abi, self);
// Note that it is really important that the type here is the right one, and matches the
// type things are read at. In case `value` is a `ScalarPair`, we don't do any magic here
// to handle padding properly, which is only correct if we never look at this data with the
@ -683,7 +688,7 @@ where
let tcx = *self.tcx;
let Some(mut alloc) = self.get_place_alloc_mut(&MPlaceTy { mplace: dest, layout })? else {
// zero-sized access
return Ok(());
return interp_ok(());
};
match value {
@ -708,7 +713,7 @@ where
alloc.write_scalar(alloc_range(Size::ZERO, a_val.size()), a_val)?;
alloc.write_scalar(alloc_range(b_offset, b_val.size()), b_val)?;
// We don't have to reset padding here, `write_immediate` will anyway do a validation run.
Ok(())
interp_ok(())
}
Immediate::Uninit => alloc.write_uninit_full(),
}
@ -729,12 +734,12 @@ where
Left(mplace) => {
let Some(mut alloc) = self.get_place_alloc_mut(&mplace)? else {
// Zero-sized access
return Ok(());
return interp_ok(());
};
alloc.write_uninit_full()?;
}
}
Ok(())
interp_ok(())
}
/// Remove all provenance in the given place.
@ -753,12 +758,12 @@ where
Left(mplace) => {
let Some(mut alloc) = self.get_place_alloc_mut(&mplace)? else {
// Zero-sized access
return Ok(());
return interp_ok(());
};
alloc.clear_provenance()?;
}
}
Ok(())
interp_ok(())
}
/// Copies the data from an operand to a place.
@ -841,7 +846,7 @@ where
)?;
}
Ok(())
interp_ok(())
}
/// Copies the data from an operand to a place.
@ -918,7 +923,7 @@ where
self.mem_copy(src.ptr(), dest.ptr(), dest_size, /*nonoverlapping*/ true)?;
self.check_misalign(src.mplace.misaligned, CheckAlignMsg::BasedOn)?;
self.check_misalign(dest.mplace.misaligned, CheckAlignMsg::BasedOn)?;
Ok(())
interp_ok(())
}
/// Ensures that a place is in memory, and returns where it is.
@ -980,7 +985,7 @@ where
Place::Ptr(mplace) => mplace,
};
// Return with the original layout and align, so that the caller can go on
Ok(MPlaceTy { mplace, layout: place.layout })
interp_ok(MPlaceTy { mplace, layout: place.layout })
}
pub fn allocate_dyn(
@ -993,7 +998,7 @@ where
span_bug!(self.cur_span(), "cannot allocate space for `extern` type, size is not known")
};
let ptr = self.allocate_ptr(size, align, kind)?;
Ok(self.ptr_with_meta_to_mplace(ptr.into(), meta, layout, /*unaligned*/ false))
interp_ok(self.ptr_with_meta_to_mplace(ptr.into(), meta, layout, /*unaligned*/ false))
}
pub fn allocate(
@ -1028,7 +1033,7 @@ where
};
let meta = Scalar::from_target_usize(u64::try_from(str.len()).unwrap(), self);
let layout = self.layout_of(self.tcx.types.str_).unwrap();
Ok(self.ptr_with_meta_to_mplace(
interp_ok(self.ptr_with_meta_to_mplace(
ptr.into(),
MemPlaceMeta::Meta(meta),
layout,
@ -1044,7 +1049,7 @@ where
let _ = self.tcx.global_alloc(raw.alloc_id);
let ptr = self.global_root_pointer(Pointer::from(raw.alloc_id))?;
let layout = self.layout_of(raw.ty)?;
Ok(self.ptr_to_mplace(ptr.into(), layout))
interp_ok(self.ptr_to_mplace(ptr.into(), layout))
}
}

20
compiler/rustc_const_eval/src/interpret/projection.rs
View file

@ -18,7 +18,7 @@ use tracing::{debug, instrument};
use super::{
InterpCx, InterpResult, MPlaceTy, Machine, MemPlaceMeta, OpTy, Provenance, Scalar, err_ub,
throw_ub, throw_unsup,
interp_ok, throw_ub, throw_unsup,
};
/// Describes the constraints placed on offset-projections.
@ -54,7 +54,7 @@ pub trait Projectable<'tcx, Prov: Provenance>: Sized + std::fmt::Debug {
// Go through the layout. There are lots of types that support a length,
// e.g., SIMD types. (But not all repr(simd) types even have FieldsShape::Array!)
match layout.fields {
abi::FieldsShape::Array { count, .. } => Ok(count),
abi::FieldsShape::Array { count, .. } => interp_ok(count),
_ => bug!("len not supported on sized type {:?}", layout.ty),
}
}
@ -115,9 +115,9 @@ impl<'a, 'tcx, Prov: Provenance, P: Projectable<'tcx, Prov>> ArrayIterator<'a, '
&mut self,
ecx: &InterpCx<'tcx, M>,
) -> InterpResult<'tcx, Option<(u64, P)>> {
let Some(idx) = self.range.next() else { return Ok(None) };
let Some(idx) = self.range.next() else { return interp_ok(None) };
// We use `Wrapping` here since the offset has already been checked when the iterator was created.
Ok(Some((
interp_ok(Some((
idx,
self.base.offset_with_meta(
self.stride * idx,
@ -258,7 +258,7 @@ where
// SIMD types must be newtypes around arrays, so all we have to do is project to their only field.
let array = self.project_field(base, 0)?;
let len = array.len(self)?;
Ok((array, len))
interp_ok((array, len))
}
fn project_constant_index<P: Projectable<'tcx, M::Provenance>>(
@ -300,7 +300,13 @@ where
debug!("project_array_fields: {base:?} {len}");
base.offset(len * stride, self.layout_of(self.tcx.types.unit).unwrap(), self)?;
// Create the iterator.
Ok(ArrayIterator { base, range: 0..len, stride, field_layout, _phantom: PhantomData })
interp_ok(ArrayIterator {
base,
range: 0..len,
stride,
field_layout,
_phantom: PhantomData,
})
}
/// Subslicing
@ -367,7 +373,7 @@ where
P: Projectable<'tcx, M::Provenance> + From<MPlaceTy<'tcx, M::Provenance>> + std::fmt::Debug,
{
use rustc_middle::mir::ProjectionElem::*;
Ok(match proj_elem {
interp_ok(match proj_elem {
OpaqueCast(ty) => {
span_bug!(self.cur_span(), "OpaqueCast({ty}) encountered after borrowck")
}

26
compiler/rustc_const_eval/src/interpret/stack.rs
View file

@ -17,7 +17,7 @@ use tracing::{info_span, instrument, trace};
use super::{
AllocId, CtfeProvenance, Immediate, InterpCx, InterpResult, MPlaceTy, Machine, MemPlace,
MemPlaceMeta, MemoryKind, Operand, Pointer, Provenance, ReturnAction, Scalar,
from_known_layout, throw_ub, throw_unsup,
from_known_layout, interp_ok, throw_ub, throw_unsup,
};
use crate::errors;
@ -189,7 +189,7 @@ impl<'tcx, Prov: Provenance> LocalState<'tcx, Prov> {
pub(super) fn access(&self) -> InterpResult<'tcx, &Operand<Prov>> {
match &self.value {
LocalValue::Dead => throw_ub!(DeadLocal), // could even be "invalid program"?
LocalValue::Live(val) => Ok(val),
LocalValue::Live(val) => interp_ok(val),
}
}
@ -199,7 +199,7 @@ impl<'tcx, Prov: Provenance> LocalState<'tcx, Prov> {
pub(super) fn access_mut(&mut self) -> InterpResult<'tcx, &mut Operand<Prov>> {
match &mut self.value {
LocalValue::Dead => throw_ub!(DeadLocal), // could even be "invalid program"?
LocalValue::Live(val) => Ok(val),
LocalValue::Live(val) => interp_ok(val),
}
}
}
@ -391,7 +391,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let span = info_span!("frame", "{}", instance);
self.frame_mut().tracing_span.enter(span);
Ok(())
interp_ok(())
}
/// Low-level helper that pops a stack frame from the stack and returns some information about
@ -426,7 +426,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
return_action = ReturnAction::NoCleanup;
};
Ok(StackPopInfo { return_action, return_to_block, return_place })
interp_ok(StackPopInfo { return_action, return_to_block, return_place })
}
/// A private helper for [`pop_stack_frame_raw`](InterpCx::pop_stack_frame_raw).
@ -449,7 +449,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
}
}
Ok(cleanup)
interp_ok(cleanup)
}
/// In the current stack frame, mark all locals as live that are not arguments and don't have
@ -464,7 +464,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
self.storage_live(local)?;
}
}
Ok(())
interp_ok(())
}
pub fn storage_live_dyn(
@ -550,7 +550,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// If the local is already live, deallocate its old memory.
let old = mem::replace(&mut self.frame_mut().locals[local].value, local_val);
self.deallocate_local(old)?;
Ok(())
interp_ok(())
}
/// Mark a storage as live, killing the previous content.
@ -566,7 +566,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// If the local is already dead, this is a NOP.
let old = mem::replace(&mut self.frame_mut().locals[local].value, LocalValue::Dead);
self.deallocate_local(old)?;
Ok(())
interp_ok(())
}
fn deallocate_local(&mut self, local: LocalValue<M::Provenance>) -> InterpResult<'tcx> {
@ -581,7 +581,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
);
self.deallocate_ptr(ptr, None, MemoryKind::Stack)?;
};
Ok(())
interp_ok(())
}
#[inline(always)]
@ -593,19 +593,19 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
) -> InterpResult<'tcx, TyAndLayout<'tcx>> {
let state = &frame.locals[local];
if let Some(layout) = state.layout.get() {
return Ok(layout);
return interp_ok(layout);
}
let layout = from_known_layout(self.tcx, self.param_env, layout, || {
let local_ty = frame.body.local_decls[local].ty;
let local_ty =
self.instantiate_from_frame_and_normalize_erasing_regions(frame, local_ty)?;
self.layout_of(local_ty)
self.layout_of(local_ty).into()
})?;
// Layouts of locals are requested a lot, so we cache them.
state.layout.set(Some(layout));
Ok(layout)
interp_ok(layout)
}
}

28
compiler/rustc_const_eval/src/interpret/step.rs
View file

@ -14,7 +14,7 @@ use tracing::{info, instrument, trace};
use super::{
FnArg, FnVal, ImmTy, Immediate, InterpCx, InterpResult, Machine, MemPlaceMeta, PlaceTy,
Projectable, Scalar, throw_ub,
Projectable, Scalar, interp_ok, throw_ub,
};
use crate::util;
@ -36,7 +36,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
#[inline(always)]
pub fn step(&mut self) -> InterpResult<'tcx, bool> {
if self.stack().is_empty() {
return Ok(false);
return interp_ok(false);
}
let Either::Left(loc) = self.frame().loc else {
@ -44,7 +44,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// Just go on unwinding.
trace!("unwinding: skipping frame");
self.return_from_current_stack_frame(/* unwinding */ true)?;
return Ok(true);
return interp_ok(true);
};
let basic_block = &self.body().basic_blocks[loc.block];
@ -55,7 +55,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
assert_eq!(old_frames, self.frame_idx());
// Advance the program counter.
self.frame_mut().loc.as_mut().left().unwrap().statement_index += 1;
return Ok(true);
return interp_ok(true);
}
M::before_terminator(self)?;
@ -67,7 +67,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
info!("// executing {:?}", loc.block);
}
}
Ok(true)
interp_ok(true)
}
/// Runs the interpretation logic for the given `mir::Statement` at the current frame and
@ -145,7 +145,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
Nop => {}
}
Ok(())
interp_ok(())
}
/// Evaluate an assignment statement.
@ -277,7 +277,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
trace!("{:?}", self.dump_place(&dest));
Ok(())
interp_ok(())
}
/// Writes the aggregate to the destination.
@ -313,7 +313,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let ptr_imm = Immediate::new_pointer_with_meta(data, meta, self);
let ptr = ImmTy::from_immediate(ptr_imm, dest.layout);
self.copy_op(&ptr, dest)?;
return Ok(());
return interp_ok(());
}
_ => (FIRST_VARIANT, dest.clone(), None),
};
@ -365,7 +365,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
)?;
}
Ok(())
interp_ok(())
}
/// Evaluate the arguments of a function call
@ -373,7 +373,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
&self,
op: &mir::Operand<'tcx>,
) -> InterpResult<'tcx, FnArg<'tcx, M::Provenance>> {
Ok(match op {
interp_ok(match op {
mir::Operand::Copy(_) | mir::Operand::Constant(_) => {
// Make a regular copy.
let op = self.eval_operand(op, None)?;
@ -442,7 +442,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
}
};
Ok(EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location })
interp_ok(EvaluatedCalleeAndArgs { callee, args, fn_sig, fn_abi, with_caller_location })
}
fn eval_terminator(&mut self, terminator: &mir::Terminator<'tcx>) -> InterpResult<'tcx> {
@ -537,7 +537,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// generic. In order to make sure that generic and non-generic code behaves
// roughly the same (and in keeping with Mir semantics) we do nothing here.
self.go_to_block(target);
return Ok(());
return interp_ok(());
}
trace!("TerminatorKind::drop: {:?}, type {}", place, place.layout.ty);
self.init_drop_in_place_call(&place, instance, target, unwind)?;
@ -566,7 +566,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// By definition, a Resume terminator means
// that we're unwinding
self.return_from_current_stack_frame(/* unwinding */ true)?;
return Ok(());
return interp_ok(());
}
// It is UB to ever encounter this.
@ -584,6 +584,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
}
}
Ok(())
interp_ok(())
}
}

14
compiler/rustc_const_eval/src/interpret/traits.rs
View file

@ -5,7 +5,9 @@ use rustc_target::abi::{Align, Size};
use tracing::trace;
use super::util::ensure_monomorphic_enough;
use super::{InterpCx, MPlaceTy, Machine, MemPlaceMeta, OffsetMode, Projectable, throw_ub};
use super::{
InterpCx, MPlaceTy, Machine, MemPlaceMeta, OffsetMode, Projectable, interp_ok, throw_ub,
};
impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
/// Creates a dynamic vtable for the given type and vtable origin. This is used only for
@ -31,7 +33,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let salt = M::get_global_alloc_salt(self, None);
let vtable_symbolic_allocation = self.tcx.reserve_and_set_vtable_alloc(ty, dyn_ty, salt);
let vtable_ptr = self.global_root_pointer(Pointer::from(vtable_symbolic_allocation))?;
Ok(vtable_ptr.into())
interp_ok(vtable_ptr.into())
}
pub fn get_vtable_size_and_align(
@ -42,7 +44,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
let ty = self.get_ptr_vtable_ty(vtable, expected_trait)?;
let layout = self.layout_of(ty)?;
assert!(layout.is_sized(), "there are no vtables for unsized types");
Ok((layout.size, layout.align.abi))
interp_ok((layout.size, layout.align.abi))
}
pub(super) fn vtable_entries(
@ -102,7 +104,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
}
}
Ok(())
interp_ok(())
}
/// Turn a place with a `dyn Trait` type into a place with the actual dynamic type.
@ -127,7 +129,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
layout,
self,
)?;
Ok(mplace)
interp_ok(mplace)
}
/// Turn a `dyn* Trait` type into an value with the actual dynamic type.
@ -147,6 +149,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
// `data` is already the right thing but has the wrong type. So we transmute it.
let layout = self.layout_of(ty)?;
let data = data.transmute(layout, self)?;
Ok(data)
interp_ok(data)
}
}

8
compiler/rustc_const_eval/src/interpret/util.rs
View file

@ -9,7 +9,7 @@ use rustc_middle::ty::{
};
use tracing::debug;
use super::{InterpCx, MPlaceTy, MemoryKind, throw_inval};
use super::{InterpCx, MPlaceTy, MemoryKind, interp_ok, throw_inval};
use crate::const_eval::{CompileTimeInterpCx, CompileTimeMachine, InterpretationResult};
/// Checks whether a type contains generic parameters which must be instantiated.
@ -23,7 +23,7 @@ where
{
debug!("ensure_monomorphic_enough: ty={:?}", ty);
if !ty.has_param() {
return Ok(());
return interp_ok(());
}
struct FoundParam;
@ -78,7 +78,7 @@ where
if matches!(ty.visit_with(&mut vis), ControlFlow::Break(FoundParam)) {
throw_inval!(TooGeneric);
} else {
Ok(())
interp_ok(())
}
}
@ -103,5 +103,5 @@ pub(crate) fn create_static_alloc<'tcx>(
assert_eq!(ecx.machine.static_root_ids, None);
ecx.machine.static_root_ids = Some((alloc_id, static_def_id));
assert!(ecx.memory.alloc_map.insert(alloc_id, (MemoryKind::Stack, alloc)).is_none());
Ok(ecx.ptr_to_mplace(Pointer::from(alloc_id).into(), layout))
interp_ok(ecx.ptr_to_mplace(Pointer::from(alloc_id).into(), layout))
}

174
compiler/rustc_const_eval/src/interpret/validity.rs
View file

@ -17,8 +17,8 @@ use rustc_hir as hir;
use rustc_middle::bug;
use rustc_middle::mir::interpret::ValidationErrorKind::{self, *};
use rustc_middle::mir::interpret::{
ExpectedKind, InterpError, InvalidMetaKind, Misalignment, PointerKind, Provenance,
UnsupportedOpInfo, ValidationErrorInfo, alloc_range,
ExpectedKind, InterpError, InterpErrorInfo, InvalidMetaKind, Misalignment, PointerKind,
Provenance, UnsupportedOpInfo, ValidationErrorInfo, alloc_range, interp_ok,
};
use rustc_middle::ty::layout::{LayoutCx, LayoutOf, TyAndLayout};
use rustc_middle::ty::{self, Ty};
@ -32,7 +32,7 @@ use super::machine::AllocMap;
use super::{
AllocId, AllocKind, CheckInAllocMsg, GlobalAlloc, ImmTy, Immediate, InterpCx, InterpResult,
MPlaceTy, Machine, MemPlaceMeta, PlaceTy, Pointer, Projectable, Scalar, ValueVisitor, err_ub,
format_interp_error, throw_ub,
format_interp_error,
};
// for the validation errors
@ -42,7 +42,7 @@ use super::InterpError::Unsupported as Unsup;
use super::UndefinedBehaviorInfo::*;
use super::UnsupportedOpInfo::*;
macro_rules! throw_validation_failure {
macro_rules! err_validation_failure {
($where:expr, $kind: expr) => {{
let where_ = &$where;
let path = if !where_.is_empty() {
@ -53,10 +53,16 @@ macro_rules! throw_validation_failure {
None
};
throw_ub!(ValidationError(ValidationErrorInfo { path, kind: $kind }))
err_ub!(ValidationError(ValidationErrorInfo { path, kind: $kind }))
}};
}
macro_rules! throw_validation_failure {
($where:expr, $kind: expr) => {
do yeet err_validation_failure!($where, $kind)
};
}
/// If $e throws an error matching the pattern, throw a validation failure.
/// Other errors are passed back to the caller, unchanged -- and if they reach the root of
/// the visitor, we make sure only validation errors and `InvalidProgram` errors are left.
@ -91,22 +97,22 @@ macro_rules! try_validation {
($e:expr, $where:expr,
$( $( $p:pat_param )|+ => $kind: expr ),+ $(,)?
) => {{
match $e {
Ok(x) => x,
$e.map_err(|e| {
// We catch the error and turn it into a validation failure. We are okay with
// allocation here as this can only slow down builds that fail anyway.
Err(e) => match e.kind() {
let (kind, backtrace) = e.into_parts();
match kind {
$(
$($p)|+ =>
throw_validation_failure!(
$($p)|+ => {
err_validation_failure!(
$where,
$kind
)
).into()
}
),+,
#[allow(unreachable_patterns)]
_ => Err::<!, _>(e)?,
_ => InterpErrorInfo::from_parts(kind, backtrace),
}
}
})?
}};
}
@ -378,7 +384,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
// Undo changes
self.path.truncate(path_len);
// Done
Ok(r)
interp_ok(r)
}
fn read_immediate(
@ -386,7 +392,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
val: &PlaceTy<'tcx, M::Provenance>,
expected: ExpectedKind,
) -> InterpResult<'tcx, ImmTy<'tcx, M::Provenance>> {
Ok(try_validation!(
interp_ok(try_validation!(
self.ecx.read_immediate(val),
self.path,
Ub(InvalidUninitBytes(None)) =>
@ -404,7 +410,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
val: &PlaceTy<'tcx, M::Provenance>,
expected: ExpectedKind,
) -> InterpResult<'tcx, Scalar<M::Provenance>> {
Ok(self.read_immediate(val, expected)?.to_scalar())
interp_ok(self.read_immediate(val, expected)?.to_scalar())
}
fn deref_pointer(
@ -469,7 +475,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
_ => bug!("Unexpected unsized type tail: {:?}", tail),
}
Ok(())
interp_ok(())
}
/// Check a reference or `Box`.
@ -510,7 +516,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
Ub(DanglingIntPointer { addr: i, .. }) => DanglingPtrNoProvenance {
ptr_kind,
// FIXME this says "null pointer" when null but we need translate
pointer: format!("{}", Pointer::<Option<AllocId>>::from_addr_invalid(*i))
pointer: format!("{}", Pointer::<Option<AllocId>>::from_addr_invalid(i))
},
Ub(PointerOutOfBounds { .. }) => DanglingPtrOutOfBounds {
ptr_kind
@ -632,7 +638,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
}
// Potentially skip recursive check.
if skip_recursive_check {
return Ok(());
return interp_ok(());
}
} else {
// This is not CTFE, so it's Miri with recursive checking.
@ -641,7 +647,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
// FIXME: should we also skip `UnsafeCell` behind shared references? Currently that is not
// needed since validation reads bypass Stacked Borrows and data race checks.
if matches!(ptr_kind, PointerKind::Box) {
return Ok(());
return interp_ok(());
}
}
let path = &self.path;
@ -654,7 +660,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
new_path
});
}
Ok(())
interp_ok(())
}
/// Check if this is a value of primitive type, and if yes check the validity of the value
@ -681,7 +687,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
self.ecx.clear_provenance(value)?;
self.add_data_range_place(value);
}
Ok(true)
interp_ok(true)
}
ty::Char => {
let scalar = self.read_scalar(value, ExpectedKind::Char)?;
@ -696,7 +702,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
self.ecx.clear_provenance(value)?;
self.add_data_range_place(value);
}
Ok(true)
interp_ok(true)
}
ty::Float(_) | ty::Int(_) | ty::Uint(_) => {
// NOTE: Keep this in sync with the array optimization for int/float
@ -713,18 +719,18 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
self.ecx.clear_provenance(value)?;
self.add_data_range_place(value);
}
Ok(true)
interp_ok(true)
}
ty::RawPtr(..) => {
let place = self.deref_pointer(value, ExpectedKind::RawPtr)?;
if place.layout.is_unsized() {
self.check_wide_ptr_meta(place.meta(), place.layout)?;
}
Ok(true)
interp_ok(true)
}
ty::Ref(_, _ty, mutbl) => {
self.check_safe_pointer(value, PointerKind::Ref(*mutbl))?;
Ok(true)
interp_ok(true)
}
ty::FnPtr(..) => {
let scalar = self.read_scalar(value, ExpectedKind::FnPtr)?;
@ -753,12 +759,12 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
}
self.add_data_range_place(value);
}
Ok(true)
interp_ok(true)
}
ty::Never => throw_validation_failure!(self.path, NeverVal),
ty::Foreign(..) | ty::FnDef(..) => {
// Nothing to check.
Ok(true)
interp_ok(true)
}
// The above should be all the primitive types. The rest is compound, we
// check them by visiting their fields/variants.
@ -771,7 +777,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
| ty::Closure(..)
| ty::Pat(..)
| ty::CoroutineClosure(..)
| ty::Coroutine(..) => Ok(false),
| ty::Coroutine(..) => interp_ok(false),
// Some types only occur during typechecking, they have no layout.
// We should not see them here and we could not check them anyway.
ty::Error(_)
@ -808,11 +814,11 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
max_value
})
} else {
return Ok(());
return interp_ok(());
}
} else if scalar_layout.is_always_valid(self.ecx) {
// Easy. (This is reachable if `enforce_number_validity` is set.)
return Ok(());
return interp_ok(());
} else {
// Conservatively, we reject, because the pointer *could* have a bad
// value.
@ -825,7 +831,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
};
// Now compare.
if valid_range.contains(bits) {
Ok(())
interp_ok(())
} else {
throw_validation_failure!(self.path, OutOfRange {
value: format!("{bits}"),
@ -884,7 +890,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
}
fn reset_padding(&mut self, place: &PlaceTy<'tcx, M::Provenance>) -> InterpResult<'tcx> {
let Some(data_bytes) = self.data_bytes.as_mut() else { return Ok(()) };
let Some(data_bytes) = self.data_bytes.as_mut() else { return interp_ok(()) };
// Our value must be in memory, otherwise we would not have set up `data_bytes`.
let mplace = self.ecx.force_allocation(place)?;
// Determine starting offset and size.
@ -896,14 +902,14 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
// If there is no padding at all, we can skip the rest: check for
// a single data range covering the entire value.
if data_bytes.0 == &[(start_offset, size)] {
return Ok(());
return interp_ok(());
}
// Get a handle for the allocation. Do this only once, to avoid looking up the same
// allocation over and over again. (Though to be fair, iterating the value already does
// exactly that.)
let Some(mut alloc) = self.ecx.get_ptr_alloc_mut(mplace.ptr(), size)? else {
// A ZST, no padding to clear.
return Ok(());
return interp_ok(());
};
// Add a "finalizer" data range at the end, so that the iteration below finds all gaps
// between ranges.
@ -930,7 +936,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValidityVisitor<'rt, 'tcx, M> {
padding_cleared_until = offset + size;
}
assert!(padding_cleared_until == start_offset + size);
Ok(())
interp_ok(())
}
/// Computes the data range of this union type:
@ -1070,7 +1076,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
val: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx, VariantIdx> {
self.with_elem(PathElem::EnumTag, move |this| {
Ok(try_validation!(
interp_ok(try_validation!(
this.ecx.read_discriminant(val),
this.path,
Ub(InvalidTag(val)) => InvalidEnumTag {
@ -1134,7 +1140,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
data_bytes.add_range(base_offset + offset, size);
}
}
Ok(())
interp_ok(())
}
#[inline]
@ -1144,7 +1150,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
val: &PlaceTy<'tcx, M::Provenance>,
) -> InterpResult<'tcx> {
self.check_safe_pointer(val, PointerKind::Box)?;
Ok(())
interp_ok(())
}
#[inline]
@ -1157,7 +1163,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
// We assume that the Scalar validity range does not restrict these values
// any further than `try_visit_primitive` does!
if self.try_visit_primitive(val)? {
return Ok(());
return interp_ok(());
}
// Special check preventing `UnsafeCell` in the inner part of constants
@ -1204,7 +1210,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
// If the size is 0, there is nothing to check.
// (`size` can only be 0 if `len` is 0, and empty arrays are always valid.)
if size == Size::ZERO {
return Ok(());
return interp_ok(());
}
// Now that we definitely have a non-ZST array, we know it lives in memory -- except it may
// be an uninitialized local variable, those are also "immediate".
@ -1224,36 +1230,33 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
// No need for an alignment check here, this is not an actual memory access.
let alloc = self.ecx.get_ptr_alloc(mplace.ptr(), size)?.expect("we already excluded size 0");
match alloc.get_bytes_strip_provenance() {
// In the happy case, we needn't check anything else.
Ok(_) => {}
alloc.get_bytes_strip_provenance().map_err(|err| {
// Some error happened, try to provide a more detailed description.
Err(err) => {
// For some errors we might be able to provide extra information.
// (This custom logic does not fit the `try_validation!` macro.)
match err.kind() {
Ub(InvalidUninitBytes(Some((_alloc_id, access)))) | Unsup(ReadPointerAsInt(Some((_alloc_id, access)))) => {
// Some byte was uninitialized, determine which
// element that byte belongs to so we can
// provide an index.
let i = usize::try_from(
access.bad.start.bytes() / layout.size.bytes(),
)
.unwrap();
self.path.push(PathElem::ArrayElem(i));
// For some errors we might be able to provide extra information.
// (This custom logic does not fit the `try_validation!` macro.)
let (kind, backtrace) = err.into_parts();
match kind {
Ub(InvalidUninitBytes(Some((_alloc_id, access)))) | Unsup(ReadPointerAsInt(Some((_alloc_id, access)))) => {
// Some byte was uninitialized, determine which
// element that byte belongs to so we can
// provide an index.
let i = usize::try_from(
access.bad.start.bytes() / layout.size.bytes(),
)
.unwrap();
self.path.push(PathElem::ArrayElem(i));
if matches!(err.kind(), Ub(InvalidUninitBytes(_))) {
throw_validation_failure!(self.path, Uninit { expected })
} else {
throw_validation_failure!(self.path, PointerAsInt { expected })
}
if matches!(kind, Ub(InvalidUninitBytes(_))) {
err_validation_failure!(self.path, Uninit { expected }).into()
} else {
err_validation_failure!(self.path, PointerAsInt { expected }).into()
}
// Propagate upwards (that will also check for unexpected errors).
_ => return Err(err),
}
// Propagate upwards (that will also check for unexpected errors).
_ => return InterpErrorInfo::from_parts(kind, backtrace),
}
}
})?;
// Don't forget that these are all non-pointer types, and thus do not preserve
// provenance.
@ -1282,7 +1285,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
// It's not great to catch errors here, since we can't give a very good path,
// but it's better than ICEing.
Ub(InvalidVTableTrait { vtable_dyn_type, expected_dyn_type }) => {
InvalidMetaWrongTrait { vtable_dyn_type, expected_dyn_type: *expected_dyn_type }
InvalidMetaWrongTrait { vtable_dyn_type, expected_dyn_type }
},
);
}
@ -1331,7 +1334,7 @@ impl<'rt, 'tcx, M: Machine<'tcx>> ValueVisitor<'tcx, M> for ValidityVisitor<'rt,
}
}
Ok(())
interp_ok(())
}
}
@ -1347,7 +1350,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
trace!("validate_operand_internal: {:?}, {:?}", *val, val.layout.ty);
// Run the visitor.
match self.run_for_validation(|ecx| {
self.run_for_validation(|ecx| {
let reset_padding = reset_provenance_and_padding && {
// Check if `val` is actually stored in memory. If not, padding is not even
// represented and we need not reset it.
@ -1363,29 +1366,22 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
};
v.visit_value(val)?;
v.reset_padding(val)?;
InterpResult::Ok(())
}) {
Ok(()) => Ok(()),
// Pass through validation failures and "invalid program" issues.
Err(err)
if matches!(
err.kind(),
err_ub!(ValidationError { .. })
| InterpError::InvalidProgram(_)
| InterpError::Unsupported(UnsupportedOpInfo::ExternTypeField)
) =>
{
Err(err)
}
// Complain about any other kind of error -- those are bad because we'd like to
// report them in a way that shows *where* in the value the issue lies.
Err(err) => {
interp_ok(())
})
.map_err(|err| {
if !matches!(
err.kind(),
err_ub!(ValidationError { .. })
| InterpError::InvalidProgram(_)
| InterpError::Unsupported(UnsupportedOpInfo::ExternTypeField)
) {
bug!(
"Unexpected error during validation: {}",
format_interp_error(self.tcx.dcx(), err)
);
}
}
err
})
}
/// This function checks the data at `op` to be const-valid.
@ -1456,6 +1452,6 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
/*reset_provenance_and_padding*/ false,
)?;
}
Ok(())
interp_ok(())
}
}

10
compiler/rustc_const_eval/src/interpret/visitor.rs
View file

@ -10,7 +10,7 @@ use rustc_middle::ty::{self, Ty};
use rustc_target::abi::{FieldIdx, FieldsShape, VariantIdx, Variants};
use tracing::trace;
use super::{InterpCx, MPlaceTy, Machine, Projectable, throw_inval};
use super::{InterpCx, MPlaceTy, Machine, Projectable, interp_ok, throw_inval};
/// How to traverse a value and what to do when we are at the leaves.
pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized {
@ -46,14 +46,14 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized {
/// Visits the given value as a union. No automatic recursion can happen here.
#[inline(always)]
fn visit_union(&mut self, _v: &Self::V, _fields: NonZero<usize>) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Visits the given value as the pointer of a `Box`. There is nothing to recurse into.
/// The type of `v` will be a raw pointer to `T`, but this is a field of `Box<T>` and the
/// pointee type is the actual `T`. `box_ty` provides the full type of the `Box` itself.
#[inline(always)]
fn visit_box(&mut self, _box_ty: Ty<'tcx>, _v: &Self::V) -> InterpResult<'tcx> {
Ok(())
interp_ok(())
}
/// Called each time we recurse down to a field of a "product-like" aggregate
@ -165,7 +165,7 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized {
self.visit_field(v, 1, &alloc)?;
// We visited all parts of this one.
return Ok(());
return interp_ok(());
}
// Non-normalized types should never show up here.
@ -222,6 +222,6 @@ pub trait ValueVisitor<'tcx, M: Machine<'tcx>>: Sized {
Variants::Single { .. } => {}
}
Ok(())
interp_ok(())
}
}

3
compiler/rustc_const_eval/src/util/check_validity_requirement.rs
View file

@ -75,7 +75,8 @@ fn check_validity_requirement_strict<'tcx>(
/*recursive*/ false,
/*reset_provenance_and_padding*/ false,
)
.is_ok())
.discard_err()
.is_some())
}
/// Implements the 'lax' (default) version of the [`check_validity_requirement`] checks; see that

2
compiler/rustc_data_structures/Cargo.toml
View file

@ -32,7 +32,7 @@ tracing = "0.1"
version = "0.12"
[target.'cfg(windows)'.dependencies.windows]
version = "0.52.0"
version = "0.57.0"
features = [
"Win32_Foundation",
"Win32_Storage_FileSystem",

2
compiler/rustc_data_structures/src/sync.rs
View file

@ -437,7 +437,7 @@ impl<T> RwLock<T> {
#[inline(always)]
pub fn leak(&self) -> &T {
let guard = self.read();
let ret = unsafe { &*std::ptr::addr_of!(*guard) };
let ret = unsafe { &*(&raw const *guard) };
std::mem::forget(guard);
ret
}

2
compiler/rustc_driver_impl/Cargo.toml
View file

@ -59,7 +59,7 @@ libc = "0.2"
# tidy-alphabetical-end
[target.'cfg(windows)'.dependencies.windows]
version = "0.52.0"
version = "0.57.0"
features = [
"Win32_System_Diagnostics_Debug",
]

2
compiler/rustc_error_codes/src/error_codes/E0013.md
View file

@ -5,7 +5,7 @@ variable cannot refer to a static variable.
Erroneous code example:
```compile_fail,E0658
```
static X: i32 = 42;
const Y: i32 = X;
```

33
compiler/rustc_error_codes/src/error_codes/E0038.md
View file

@ -5,9 +5,9 @@ trait, written in type positions) but this was a bit too confusing, so we now
write `dyn Trait`.
Some traits are not allowed to be used as trait object types. The traits that
are allowed to be used as trait object types are called "object-safe" traits.
Attempting to use a trait object type for a trait that is not object-safe will
trigger error E0038.
are allowed to be used as trait object types are called "dyn-compatible"[^1]
traits. Attempting to use a trait object type for a trait that is not
dyn-compatible will trigger error E0038.
Two general aspects of trait object types give rise to the restrictions:
@ -25,13 +25,16 @@ Two general aspects of trait object types give rise to the restrictions:
objects with the same trait object type may point to vtables from different
implementations.
The specific conditions that violate object-safety follow, most of which relate
to missing size information and vtable polymorphism arising from these aspects.
The specific conditions that violate dyn-compatibility follow, most of which
relate to missing size information and vtable polymorphism arising from these
aspects.
[^1]: Formerly known as "object-safe".
### The trait requires `Self: Sized`
Traits that are declared as `Trait: Sized` or which otherwise inherit a
constraint of `Self:Sized` are not object-safe.
constraint of `Self:Sized` are not dyn-compatible.
The reasoning behind this is somewhat subtle. It derives from the fact that Rust
requires (and defines) that every trait object type `dyn Trait` automatically
@ -58,7 +61,7 @@ implement a sized trait like `Trait:Sized`. So, rather than allow an exception
to the rule that `dyn Trait` always implements `Trait`, Rust chooses to prohibit
such a `dyn Trait` from existing at all.
Only unsized traits are considered object-safe.
Only unsized traits are considered dyn-compatible.
Generally, `Self: Sized` is used to indicate that the trait should not be used
as a trait object. If the trait comes from your own crate, consider removing
@ -103,8 +106,8 @@ fn call_foo(x: Box<dyn Trait>) {
}
```
If only some methods aren't object-safe, you can add a `where Self: Sized` bound
on them to mark them as explicitly unavailable to trait objects. The
If only some methods aren't dyn-compatible, you can add a `where Self: Sized`
bound on them to mark them as explicitly unavailable to trait objects. The
functionality will still be available to all other implementers, including
`Box<dyn Trait>` which is itself sized (assuming you `impl Trait for Box<dyn
Trait>`).
@ -117,7 +120,7 @@ trait Trait {
```
Now, `foo()` can no longer be called on a trait object, but you will now be
allowed to make a trait object, and that will be able to call any object-safe
allowed to make a trait object, and that will be able to call any dyn-compatible
methods. With such a bound, one can still call `foo()` on types implementing
that trait that aren't behind trait objects.
@ -306,7 +309,7 @@ Here, the supertrait might have methods as follows:
```
trait Super<A: ?Sized> {
fn get_a(&self) -> &A; // note that this is object safe!
fn get_a(&self) -> &A; // note that this is dyn-compatible!
}
```
@ -314,10 +317,10 @@ If the trait `Trait` was deriving from something like `Super<String>` or
`Super<T>` (where `Foo` itself is `Foo<T>`), this is okay, because given a type
`get_a()` will definitely return an object of that type.
However, if it derives from `Super<Self>`, even though `Super` is object safe,
the method `get_a()` would return an object of unknown type when called on the
function. `Self` type parameters let us make object safe traits no longer safe,
so they are forbidden when specifying supertraits.
However, if it derives from `Super<Self>`, even though `Super` is
dyn-compatible, the method `get_a()` would return an object of unknown type when
called on the function. `Self` type parameters let us make dyn-compatible traits
no longer compatible, so they are forbidden when specifying supertraits.
There's no easy fix for this. Generally, code will need to be refactored so that
you no longer need to derive from `Super<Self>`.

Some files were not shown because too many files have changed in this diff Show more