user0/rust
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Auto merge of #146765 - Zalathar:rollup-ewh4s9o, r=Zalathar

Browse source 
Rollup of 10 pull requests

Successful merges:

 - rust-lang/rust#146229 (Automatically switch to lto-fat when flag RUSTFLAGS="- Zautodiff=Enable" is set)
 - rust-lang/rust#146484 (rustdoc-search: JavaScript optimization based on Firefox Profiler output)
 - rust-lang/rust#146541 (std: simplify host lookup)
 - rust-lang/rust#146615 (rustc_codegen_llvm: Feature Conversion Tidying)
 - rust-lang/rust#146638 (`rustc_next_trait_solver`: canonical out of `EvalCtxt`)
 - rust-lang/rust#146663 (Allow windows resource compiler to be overridden)
 - rust-lang/rust#146691 (std: Fix WASI implementation of `remove_dir_all`)
 - rust-lang/rust#146709 (stdarch subtree update)
 - rust-lang/rust#146738 (Fix tidy spellchecking on Windows)
 - rust-lang/rust#146740 (miri subtree update)

r? `@ghost`
`@rustbot` modify labels: rollup
This commit is contained in:
bors 2025-09-19 15:22:22 +00:00
commit 59043567a5
248 changed files with 10343 additions and 13062 deletions
Download patch file Download diff file Expand all files Collapse all files

37
Cargo.lock
View file

@ -674,7 +674,7 @@ checksum = "fe6d2e5af09e8c8ad56c969f2157a3d4238cebc7c55f0a517728c38f7b200f81"
dependencies = [
"serde",
"termcolor",
"unicode-width 0.1.14",
"unicode-width 0.2.1",
]
[[package]]
@ -937,23 +937,24 @@ dependencies = [
[[package]]
name = "cxx"
version = "1.0.168"
version = "1.0.185"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7aa144b12f11741f0dab5b4182896afad46faa0598b6a061f7b9d17a21837ba7"
checksum = "2f81de88da10862f22b5b3a60f18f6f42bbe7cb8faa24845dd7b1e4e22190e77"
dependencies = [
"cc",
"cxx-build",
"cxxbridge-cmd",
"cxxbridge-flags",
"cxxbridge-macro",
"foldhash",
"foldhash 0.2.0",
"link-cplusplus",
]
[[package]]
name = "cxx-build"
version = "1.0.168"
version = "1.0.185"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12d3cbb84fb003242941c231b45ca9417e786e66e94baa39584bd99df3a270b6"
checksum = "5edd58bf75c3fdfc80d79806403af626570662f7b6cc782a7fabe156166bd6d6"
dependencies = [
"cc",
"codespan-reporting",
@ -966,9 +967,9 @@ dependencies = [
[[package]]
name = "cxxbridge-cmd"
version = "1.0.168"
version = "1.0.185"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fa36b7b249d43f67a3f54bd65788e35e7afe64bbc671396387a48b3e8aaea94"
checksum = "fd46bf2b541a4e0c2d5abba76607379ee05d68e714868e3cb406dc8d591ce2d2"
dependencies = [
"clap",
"codespan-reporting",
@ -980,15 +981,15 @@ dependencies = [
[[package]]
name = "cxxbridge-flags"
version = "1.0.168"
version = "1.0.185"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77707c70f6563edc5429618ca34a07241b75ebab35bd01d46697c75d58f8ddfe"
checksum = "2c79b68f6a3a8f809d39b38ae8af61305a6113819b19b262643b9c21353b92d9"
[[package]]
name = "cxxbridge-macro"
version = "1.0.168"
version = "1.0.185"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ede6c0fb7e318f0a11799b86ee29dcf17b9be2960bd379a6c38e1a96a6010fff"
checksum = "862b7fdb048ff9ef0779a0d0a03affd09746c4c875543746b640756be9cff2af"
dependencies = [
"indexmap",
"proc-macro2",
@ -1388,6 +1389,12 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
[[package]]
name = "foldhash"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "form_urlencoded"
version = "1.2.1"
@ -1567,7 +1574,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
"allocator-api2",
"equivalent",
"foldhash",
"foldhash 0.1.5",
"serde",
]
@ -2160,9 +2167,9 @@ dependencies = [
[[package]]
name = "link-cplusplus"
version = "1.0.10"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a6f6da007f968f9def0d65a05b187e2960183de70c160204ecfccf0ee330212"
checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82"
dependencies = [
"cc",
]

3
bootstrap.example.toml
View file

@ -325,6 +325,9 @@
# Defaults to the Python interpreter used to execute x.py.
#build.python = "python"
# The path to (or name of) the resource compiler executable to use on Windows.
#build.windows-rc = "rc.exe"
# The path to the REUSE executable to use. Note that REUSE is not required in
# most cases, as our tooling relies on a cached (and shrunk) copy of the
# REUSE output present in the git repository and in our source tarballs.

42
compiler/rustc_codegen_llvm/src/llvm_util.rs
View file

@ -217,27 +217,16 @@ impl<'a> IntoIterator for LLVMFeature<'a> {
/// Rust can also be build with an external precompiled version of LLVM which might lead to failures
/// if the oldest tested / supported LLVM version doesn't yet support the relevant intrinsics.
pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFeature<'a>> {
let arch = if sess.target.arch == "x86_64" {
"x86"
} else if sess.target.arch == "arm64ec" {
"aarch64"
} else if sess.target.arch == "sparc64" {
"sparc"
} else if sess.target.arch == "powerpc64" {
"powerpc"
} else {
&*sess.target.arch
let raw_arch = &*sess.target.arch;
let arch = match raw_arch {
"x86_64" => "x86",
"arm64ec" => "aarch64",
"sparc64" => "sparc",
"powerpc64" => "powerpc",
_ => raw_arch,
};
let (major, _, _) = get_version();
match (arch, s) {
("x86", "sse4.2") => Some(LLVMFeature::with_dependencies(
"sse4.2",
smallvec![TargetFeatureFoldStrength::EnableOnly("crc32")],
)),
("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")),
("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")),
("x86", "bmi1") => Some(LLVMFeature::new("bmi")),
("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")),
("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")),
("aarch64", "rcpc2") => Some(LLVMFeature::new("rcpc-immo")),
("aarch64", "dpb") => Some(LLVMFeature::new("ccpp")),
("aarch64", "dpb2") => Some(LLVMFeature::new("ccdp")),
@ -260,14 +249,23 @@ pub(crate) fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> Option<LLVMFea
("aarch64", "fpmr") => None, // only existed in 18
("arm", "fp16") => Some(LLVMFeature::new("fullfp16")),
// Filter out features that are not supported by the current LLVM version
("loongarch32" | "loongarch64", "32s") if get_version().0 < 21 => None,
("loongarch32" | "loongarch64", "32s") if major < 21 => None,
("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")),
("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")),
("x86", "sse4.2") => Some(LLVMFeature::with_dependencies(
"sse4.2",
smallvec![TargetFeatureFoldStrength::EnableOnly("crc32")],
)),
("x86", "pclmulqdq") => Some(LLVMFeature::new("pclmul")),
("x86", "rdrand") => Some(LLVMFeature::new("rdrnd")),
("x86", "bmi1") => Some(LLVMFeature::new("bmi")),
("x86", "cmpxchg16b") => Some(LLVMFeature::new("cx16")),
("x86", "lahfsahf") => Some(LLVMFeature::new("sahf")),
// Enable the evex512 target feature if an avx512 target feature is enabled.
("x86", s) if s.starts_with("avx512") => Some(LLVMFeature::with_dependencies(
s,
smallvec![TargetFeatureFoldStrength::EnableOnly("evex512")],
)),
("sparc", "leoncasa") => Some(LLVMFeature::new("hasleoncasa")),
("powerpc", "power8-crypto") => Some(LLVMFeature::new("crypto")),
("x86", "avx10.1") => Some(LLVMFeature::new("avx10.1-512")),
("x86", "avx10.2") => Some(LLVMFeature::new("avx10.2-512")),
("x86", "apxf") => Some(LLVMFeature::with_dependencies(

2
compiler/rustc_codegen_ssa/messages.ftl
View file

@ -8,8 +8,6 @@ codegen_ssa_aix_strip_not_used = using host's `strip` binary to cross-compile to
codegen_ssa_archive_build_failure = failed to build archive at `{$path}`: {$error}
codegen_ssa_autodiff_without_lto = using the autodiff feature requires using fat-lto
codegen_ssa_bare_instruction_set = `#[instruction_set]` requires an argument
codegen_ssa_binary_output_to_tty = option `-o` or `--emit` is used to write binary output type `{$shorthand}` to stdout, but stdout is a tty

4
compiler/rustc_codegen_ssa/src/errors.rs
View file

@ -37,10 +37,6 @@ pub(crate) struct CguNotRecorded<'a> {
pub cgu_name: &'a str,
}
#[derive(Diagnostic)]
#[diag(codegen_ssa_autodiff_without_lto)]
pub struct AutodiffWithoutLto;
#[derive(Diagnostic)]
#[diag(codegen_ssa_unknown_reuse_kind)]
pub(crate) struct UnknownReuseKind {

7
compiler/rustc_next_trait_solver/src/canonicalizer.rs → compiler/rustc_next_trait_solver/src/canonical/canonicalizer.rs
View file

@ -57,7 +57,7 @@ enum CanonicalizeMode {
},
}
pub struct Canonicalizer<'a, D: SolverDelegate<Interner = I>, I: Interner> {
pub(super) struct Canonicalizer<'a, D: SolverDelegate<Interner = I>, I: Interner> {
delegate: &'a D,
// Immutable field.
@ -83,7 +83,7 @@ pub struct Canonicalizer<'a, D: SolverDelegate<Interner = I>, I: Interner> {
}
impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
pub fn canonicalize_response<T: TypeFoldable<I>>(
pub(super) fn canonicalize_response<T: TypeFoldable<I>>(
delegate: &'a D,
max_input_universe: ty::UniverseIndex,
variables: &'a mut Vec<I::GenericArg>,
@ -112,7 +112,6 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
let (max_universe, variables) = canonicalizer.finalize();
Canonical { max_universe, variables, value }
}
fn canonicalize_param_env(
delegate: &'a D,
variables: &'a mut Vec<I::GenericArg>,
@ -195,7 +194,7 @@ impl<'a, D: SolverDelegate<Interner = I>, I: Interner> Canonicalizer<'a, D, I> {
///
/// We want to keep the option of canonicalizing `'static` to an existential
/// variable in the future by changing the way we detect global where-bounds.
pub fn canonicalize_input<P: TypeFoldable<I>>(
pub(super) fn canonicalize_input<P: TypeFoldable<I>>(
delegate: &'a D,
variables: &'a mut Vec<I::GenericArg>,
input: QueryInput<I, P>,

364
compiler/rustc_next_trait_solver/src/canonical/mod.rs Normal file
View file

@ -0,0 +1,364 @@
//! Canonicalization is used to separate some goal from its context,
//! throwing away unnecessary information in the process.
//!
//! This is necessary to cache goals containing inference variables
//! and placeholders without restricting them to the current `InferCtxt`.
//!
//! Canonicalization is fairly involved, for more details see the relevant
//! section of the [rustc-dev-guide][c].
//!
//! [c]: https://rustc-dev-guide.rust-lang.org/solve/canonicalization.html
use std::iter;
use canonicalizer::Canonicalizer;
use rustc_index::IndexVec;
use rustc_type_ir::inherent::*;
use rustc_type_ir::relate::solver_relating::RelateExt;
use rustc_type_ir::{
self as ty, Canonical, CanonicalVarKind, CanonicalVarValues, InferCtxtLike, Interner,
TypeFoldable,
};
use tracing::instrument;
use crate::delegate::SolverDelegate;
use crate::resolve::eager_resolve_vars;
use crate::solve::{
CanonicalInput, CanonicalResponse, Certainty, ExternalConstraintsData, Goal,
NestedNormalizationGoals, PredefinedOpaquesData, QueryInput, Response, inspect,
};
pub mod canonicalizer;
trait ResponseT<I: Interner> {
fn var_values(&self) -> CanonicalVarValues<I>;
}
impl<I: Interner> ResponseT<I> for Response<I> {
fn var_values(&self) -> CanonicalVarValues<I> {
self.var_values
}
}
impl<I: Interner, T> ResponseT<I> for inspect::State<I, T> {
fn var_values(&self) -> CanonicalVarValues<I> {
self.var_values
}
}
/// Canonicalizes the goal remembering the original values
/// for each bound variable.
///
/// This expects `goal` and `opaque_types` to be eager resolved.
pub(super) fn canonicalize_goal<D, I>(
delegate: &D,
goal: Goal<I, I::Predicate>,
opaque_types: Vec<(ty::OpaqueTypeKey<I>, I::Ty)>,
) -> (Vec<I::GenericArg>, CanonicalInput<I, I::Predicate>)
where
D: SolverDelegate<Interner = I>,
I: Interner,
{
let mut orig_values = Default::default();
let canonical = Canonicalizer::canonicalize_input(
delegate,
&mut orig_values,
QueryInput {
goal,
predefined_opaques_in_body: delegate
.cx()
.mk_predefined_opaques_in_body(PredefinedOpaquesData { opaque_types }),
},
);
let query_input = ty::CanonicalQueryInput { canonical, typing_mode: delegate.typing_mode() };
(orig_values, query_input)
}
pub(super) fn canonicalize_response<D, I, T>(
delegate: &D,
max_input_universe: ty::UniverseIndex,
value: T,
) -> ty::Canonical<I, T>
where
D: SolverDelegate<Interner = I>,
I: Interner,
T: TypeFoldable<I>,
{
let mut orig_values = Default::default();
let canonical =
Canonicalizer::canonicalize_response(delegate, max_input_universe, &mut orig_values, value);
canonical
}
/// After calling a canonical query, we apply the constraints returned
/// by the query using this function.
///
/// This happens in three steps:
/// - we instantiate the bound variables of the query response
/// - we unify the `var_values` of the response with the `original_values`
/// - we apply the `external_constraints` returned by the query, returning
/// the `normalization_nested_goals`
pub(super) fn instantiate_and_apply_query_response<D, I>(
delegate: &D,
param_env: I::ParamEnv,
original_values: &[I::GenericArg],
response: CanonicalResponse<I>,
span: I::Span,
) -> (NestedNormalizationGoals<I>, Certainty)
where
D: SolverDelegate<Interner = I>,
I: Interner,
{
let instantiation =
compute_query_response_instantiation_values(delegate, &original_values, &response, span);
let Response { var_values, external_constraints, certainty } =
delegate.instantiate_canonical(response, instantiation);
unify_query_var_values(delegate, param_env, &original_values, var_values, span);
let ExternalConstraintsData { region_constraints, opaque_types, normalization_nested_goals } =
&*external_constraints;
register_region_constraints(delegate, region_constraints, span);
register_new_opaque_types(delegate, opaque_types, span);
(normalization_nested_goals.clone(), certainty)
}
/// This returns the canonical variable values to instantiate the bound variables of
/// the canonical response. This depends on the `original_values` for the
/// bound variables.
fn compute_query_response_instantiation_values<D, I, T>(
delegate: &D,
original_values: &[I::GenericArg],
response: &Canonical<I, T>,
span: I::Span,
) -> CanonicalVarValues<I>
where
D: SolverDelegate<Interner = I>,
I: Interner,
T: ResponseT<I>,
{
// FIXME: Longterm canonical queries should deal with all placeholders
// created inside of the query directly instead of returning them to the
// caller.
let prev_universe = delegate.universe();
let universes_created_in_query = response.max_universe.index();
for _ in 0..universes_created_in_query {
delegate.create_next_universe();
}
let var_values = response.value.var_values();
assert_eq!(original_values.len(), var_values.len());
// If the query did not make progress with constraining inference variables,
// we would normally create a new inference variables for bound existential variables
// only then unify this new inference variable with the inference variable from
// the input.
//
// We therefore instantiate the existential variable in the canonical response with the
// inference variable of the input right away, which is more performant.
let mut opt_values = IndexVec::from_elem_n(None, response.variables.len());
for (original_value, result_value) in iter::zip(original_values, var_values.var_values.iter()) {
match result_value.kind() {
ty::GenericArgKind::Type(t) => {
// We disable the instantiation guess for inference variables
// and only use it for placeholders. We need to handle the
// `sub_root` of type inference variables which would make this
// more involved. They are also a lot rarer than region variables.
if let ty::Bound(debruijn, b) = t.kind()
&& !matches!(
response.variables.get(b.var().as_usize()).unwrap(),
CanonicalVarKind::Ty { .. }
)
{
assert_eq!(debruijn, ty::INNERMOST);
opt_values[b.var()] = Some(*original_value);
}
}
ty::GenericArgKind::Lifetime(r) => {
if let ty::ReBound(debruijn, br) = r.kind() {
assert_eq!(debruijn, ty::INNERMOST);
opt_values[br.var()] = Some(*original_value);
}
}
ty::GenericArgKind::Const(c) => {
if let ty::ConstKind::Bound(debruijn, bv) = c.kind() {
assert_eq!(debruijn, ty::INNERMOST);
opt_values[bv.var()] = Some(*original_value);
}
}
}
}
CanonicalVarValues::instantiate(delegate.cx(), response.variables, |var_values, kind| {
if kind.universe() != ty::UniverseIndex::ROOT {
// A variable from inside a binder of the query. While ideally these shouldn't
// exist at all (see the FIXME at the start of this method), we have to deal with
// them for now.
delegate.instantiate_canonical_var(kind, span, &var_values, |idx| {
prev_universe + idx.index()
})
} else if kind.is_existential() {
// As an optimization we sometimes avoid creating a new inference variable here.
//
// All new inference variables we create start out in the current universe of the caller.
// This is conceptually wrong as these inference variables would be able to name
// more placeholders then they should be able to. However the inference variables have
// to "come from somewhere", so by equating them with the original values of the caller
// later on, we pull them down into their correct universe again.
if let Some(v) = opt_values[ty::BoundVar::from_usize(var_values.len())] {
v
} else {
delegate.instantiate_canonical_var(kind, span, &var_values, |_| prev_universe)
}
} else {
// For placeholders which were already part of the input, we simply map this
// universal bound variable back the placeholder of the input.
original_values[kind.expect_placeholder_index()]
}
})
}
/// Unify the `original_values` with the `var_values` returned by the canonical query..
///
/// This assumes that this unification will always succeed. This is the case when
/// applying a query response right away. However, calling a canonical query, doing any
/// other kind of trait solving, and only then instantiating the result of the query
/// can cause the instantiation to fail. This is not supported and we ICE in this case.
///
/// We always structurally instantiate aliases. Relating aliases needs to be different
/// depending on whether the alias is *rigid* or not. We're only really able to tell
/// whether an alias is rigid by using the trait solver. When instantiating a response
/// from the solver we assume that the solver correctly handled aliases and therefore
/// always relate them structurally here.
#[instrument(level = "trace", skip(delegate))]
fn unify_query_var_values<D, I>(
delegate: &D,
param_env: I::ParamEnv,
original_values: &[I::GenericArg],
var_values: CanonicalVarValues<I>,
span: I::Span,
) where
D: SolverDelegate<Interner = I>,
I: Interner,
{
assert_eq!(original_values.len(), var_values.len());
for (&orig, response) in iter::zip(original_values, var_values.var_values.iter()) {
let goals =
delegate.eq_structurally_relating_aliases(param_env, orig, response, span).unwrap();
assert!(goals.is_empty());
}
}
fn register_region_constraints<D, I>(
delegate: &D,
outlives: &[ty::OutlivesPredicate<I, I::GenericArg>],
span: I::Span,
) where
D: SolverDelegate<Interner = I>,
I: Interner,
{
for &ty::OutlivesPredicate(lhs, rhs) in outlives {
match lhs.kind() {
ty::GenericArgKind::Lifetime(lhs) => delegate.sub_regions(rhs, lhs, span),
ty::GenericArgKind::Type(lhs) => delegate.register_ty_outlives(lhs, rhs, span),
ty::GenericArgKind::Const(_) => panic!("const outlives: {lhs:?}: {rhs:?}"),
}
}
}
fn register_new_opaque_types<D, I>(
delegate: &D,
opaque_types: &[(ty::OpaqueTypeKey<I>, I::Ty)],
span: I::Span,
) where
D: SolverDelegate<Interner = I>,
I: Interner,
{
for &(key, ty) in opaque_types {
let prev = delegate.register_hidden_type_in_storage(key, ty, span);
// We eagerly resolve inference variables when computing the query response.
// This can cause previously distinct opaque type keys to now be structurally equal.
//
// To handle this, we store any duplicate entries in a separate list to check them
// at the end of typeck/borrowck. We could alternatively eagerly equate the hidden
// types here. However, doing so is difficult as it may result in nested goals and
// any errors may make it harder to track the control flow for diagnostics.
if let Some(prev) = prev {
delegate.add_duplicate_opaque_type(key, prev, span);
}
}
}
/// Used by proof trees to be able to recompute intermediate actions while
/// evaluating a goal. The `var_values` not only include the bound variables
/// of the query input, but also contain all unconstrained inference vars
/// created while evaluating this goal.
pub fn make_canonical_state<D, I, T>(
delegate: &D,
var_values: &[I::GenericArg],
max_input_universe: ty::UniverseIndex,
data: T,
) -> inspect::CanonicalState<I, T>
where
D: SolverDelegate<Interner = I>,
I: Interner,
T: TypeFoldable<I>,
{
let var_values = CanonicalVarValues { var_values: delegate.cx().mk_args(var_values) };
let state = inspect::State { var_values, data };
let state = eager_resolve_vars(delegate, state);
Canonicalizer::canonicalize_response(delegate, max_input_universe, &mut vec![], state)
}
// FIXME: needs to be pub to be accessed by downstream
// `rustc_trait_selection::solve::inspect::analyse`.
pub fn instantiate_canonical_state<D, I, T>(
delegate: &D,
span: I::Span,
param_env: I::ParamEnv,
orig_values: &mut Vec<I::GenericArg>,
state: inspect::CanonicalState<I, T>,
) -> T
where
D: SolverDelegate<Interner = I>,
I: Interner,
T: TypeFoldable<I>,
{
// In case any fresh inference variables have been created between `state`
// and the previous instantiation, extend `orig_values` for it.
orig_values.extend(
state.value.var_values.var_values.as_slice()[orig_values.len()..]
.iter()
.map(|&arg| delegate.fresh_var_for_kind_with_span(arg, span)),
);
let instantiation =
compute_query_response_instantiation_values(delegate, orig_values, &state, span);
let inspect::State { var_values, data } = delegate.instantiate_canonical(state, instantiation);
unify_query_var_values(delegate, param_env, orig_values, var_values, span);
data
}
pub fn response_no_constraints_raw<I: Interner>(
cx: I,
max_universe: ty::UniverseIndex,
variables: I::CanonicalVarKinds,
certainty: Certainty,
) -> CanonicalResponse<I> {
ty::Canonical {
max_universe,
variables,
value: Response {
var_values: ty::CanonicalVarValues::make_identity(cx, variables),
// FIXME: maybe we should store the "no response" version in cx, like
// we do for cx.types and stuff.
external_constraints: cx.mk_external_constraints(ExternalConstraintsData::default()),
certainty,
},
}
}

2
compiler/rustc_next_trait_solver/src/lib.rs
View file

@ -10,7 +10,7 @@
#![allow(rustc::usage_of_type_ir_traits)]
// tidy-alphabetical-end
pub mod canonicalizer;
pub mod canonical;
pub mod coherence;
pub mod delegate;
pub mod placeholder;

517
compiler/rustc_next_trait_solver/src/solve/eval_ctxt/canonical.rs
View file

@ -1,517 +0,0 @@
//! Canonicalization is used to separate some goal from its context,
//! throwing away unnecessary information in the process.
//!
//! This is necessary to cache goals containing inference variables
//! and placeholders without restricting them to the current `InferCtxt`.
//!
//! Canonicalization is fairly involved, for more details see the relevant
//! section of the [rustc-dev-guide][c].
//!
//! [c]: https://rustc-dev-guide.rust-lang.org/solve/canonicalization.html
use std::iter;
use rustc_index::IndexVec;
use rustc_type_ir::data_structures::HashSet;
use rustc_type_ir::inherent::*;
use rustc_type_ir::relate::solver_relating::RelateExt;
use rustc_type_ir::solve::OpaqueTypesJank;
use rustc_type_ir::{
self as ty, Canonical, CanonicalVarKind, CanonicalVarValues, InferCtxtLike, Interner,
TypeFoldable,
};
use tracing::{debug, instrument, trace};
use crate::canonicalizer::Canonicalizer;
use crate::delegate::SolverDelegate;
use crate::resolve::eager_resolve_vars;
use crate::solve::eval_ctxt::CurrentGoalKind;
use crate::solve::{
CanonicalInput, CanonicalResponse, Certainty, EvalCtxt, ExternalConstraintsData, Goal,
MaybeCause, NestedNormalizationGoals, NoSolution, PredefinedOpaquesData, QueryInput,
QueryResult, Response, inspect, response_no_constraints_raw,
};
trait ResponseT<I: Interner> {
fn var_values(&self) -> CanonicalVarValues<I>;
}
impl<I: Interner> ResponseT<I> for Response<I> {
fn var_values(&self) -> CanonicalVarValues<I> {
self.var_values
}
}
impl<I: Interner, T> ResponseT<I> for inspect::State<I, T> {
fn var_values(&self) -> CanonicalVarValues<I> {
self.var_values
}
}
impl<D, I> EvalCtxt<'_, D>
where
D: SolverDelegate<Interner = I>,
I: Interner,
{
/// Canonicalizes the goal remembering the original values
/// for each bound variable.
///
/// This expects `goal` and `opaque_types` to be eager resolved.
pub(super) fn canonicalize_goal(
delegate: &D,
goal: Goal<I, I::Predicate>,
opaque_types: Vec<(ty::OpaqueTypeKey<I>, I::Ty)>,
) -> (Vec<I::GenericArg>, CanonicalInput<I, I::Predicate>) {
let mut orig_values = Default::default();
let canonical = Canonicalizer::canonicalize_input(
delegate,
&mut orig_values,
QueryInput {
goal,
predefined_opaques_in_body: delegate
.cx()
.mk_predefined_opaques_in_body(PredefinedOpaquesData { opaque_types }),
},
);
let query_input =
ty::CanonicalQueryInput { canonical, typing_mode: delegate.typing_mode() };
(orig_values, query_input)
}
/// To return the constraints of a canonical query to the caller, we canonicalize:
///
/// - `var_values`: a map from bound variables in the canonical goal to
/// the values inferred while solving the instantiated goal.
/// - `external_constraints`: additional constraints which aren't expressible
/// using simple unification of inference variables.
///
/// This takes the `shallow_certainty` which represents whether we're confident
/// that the final result of the current goal only depends on the nested goals.
///
/// In case this is `Certainty::Maybe`, there may still be additional nested goals
/// or inference constraints required for this candidate to be hold. The candidate
/// always requires all already added constraints and nested goals.
#[instrument(level = "trace", skip(self), ret)]
pub(in crate::solve) fn evaluate_added_goals_and_make_canonical_response(
&mut self,
shallow_certainty: Certainty,
) -> QueryResult<I> {
self.inspect.make_canonical_response(shallow_certainty);
let goals_certainty = self.try_evaluate_added_goals()?;
assert_eq!(
self.tainted,
Ok(()),
"EvalCtxt is tainted -- nested goals may have been dropped in a \
previous call to `try_evaluate_added_goals!`"
);
// We only check for leaks from universes which were entered inside
// of the query.
self.delegate.leak_check(self.max_input_universe).map_err(|NoSolution| {
trace!("failed the leak check");
NoSolution
})?;
let (certainty, normalization_nested_goals) =
match (self.current_goal_kind, shallow_certainty) {
// When normalizing, we've replaced the expected term with an unconstrained
// inference variable. This means that we dropped information which could
// have been important. We handle this by instead returning the nested goals
// to the caller, where they are then handled. We only do so if we do not
// need to recompute the `NormalizesTo` goal afterwards to avoid repeatedly
// uplifting its nested goals. This is the case if the `shallow_certainty` is
// `Certainty::Yes`.
(CurrentGoalKind::NormalizesTo, Certainty::Yes) => {
let goals = std::mem::take(&mut self.nested_goals);
// As we return all ambiguous nested goals, we can ignore the certainty
// returned by `self.try_evaluate_added_goals()`.
if goals.is_empty() {
assert!(matches!(goals_certainty, Certainty::Yes));
}
(
Certainty::Yes,
NestedNormalizationGoals(
goals.into_iter().map(|(s, g, _)| (s, g)).collect(),
),
)
}
_ => {
let certainty = shallow_certainty.and(goals_certainty);
(certainty, NestedNormalizationGoals::empty())
}
};
if let Certainty::Maybe {
cause: cause @ MaybeCause::Overflow { keep_constraints: false, .. },
opaque_types_jank,
} = certainty
{
// If we have overflow, it's probable that we're substituting a type
// into itself infinitely and any partial substitutions in the query
// response are probably not useful anyways, so just return an empty
// query response.
//
// This may prevent us from potentially useful inference, e.g.
// 2 candidates, one ambiguous and one overflow, which both
// have the same inference constraints.
//
// Changing this to retain some constraints in the future
// won't be a breaking change, so this is good enough for now.
return Ok(self.make_ambiguous_response_no_constraints(cause, opaque_types_jank));
}
let external_constraints =
self.compute_external_query_constraints(certainty, normalization_nested_goals);
let (var_values, mut external_constraints) =
eager_resolve_vars(self.delegate, (self.var_values, external_constraints));
// Remove any trivial or duplicated region constraints once we've resolved regions
let mut unique = HashSet::default();
external_constraints.region_constraints.retain(|outlives| {
outlives.0.as_region().is_none_or(|re| re != outlives.1) && unique.insert(*outlives)
});
let canonical = Canonicalizer::canonicalize_response(
self.delegate,
self.max_input_universe,
&mut Default::default(),
Response {
var_values,
certainty,
external_constraints: self.cx().mk_external_constraints(external_constraints),
},
);
// HACK: We bail with overflow if the response would have too many non-region
// inference variables. This tends to only happen if we encounter a lot of
// ambiguous alias types which get replaced with fresh inference variables
// during generalization. This prevents hangs caused by an exponential blowup,
// see tests/ui/traits/next-solver/coherence-alias-hang.rs.
match self.current_goal_kind {
// We don't do so for `NormalizesTo` goals as we erased the expected term and
// bailing with overflow here would prevent us from detecting a type-mismatch,
// causing a coherence error in diesel, see #131969. We still bail with overflow
// when later returning from the parent AliasRelate goal.
CurrentGoalKind::NormalizesTo => {}
CurrentGoalKind::Misc | CurrentGoalKind::CoinductiveTrait => {
let num_non_region_vars = canonical
.variables
.iter()
.filter(|c| !c.is_region() && c.is_existential())
.count();
if num_non_region_vars > self.cx().recursion_limit() {
debug!(?num_non_region_vars, "too many inference variables -> overflow");
return Ok(self.make_ambiguous_response_no_constraints(
MaybeCause::Overflow {
suggest_increasing_limit: true,
keep_constraints: false,
},
OpaqueTypesJank::AllGood,
));
}
}
}
Ok(canonical)
}
/// Constructs a totally unconstrained, ambiguous response to a goal.
///
/// Take care when using this, since often it's useful to respond with
/// ambiguity but return constrained variables to guide inference.
pub(in crate::solve) fn make_ambiguous_response_no_constraints(
&self,
cause: MaybeCause,
opaque_types_jank: OpaqueTypesJank,
) -> CanonicalResponse<I> {
response_no_constraints_raw(
self.cx(),
self.max_input_universe,
self.variables,
Certainty::Maybe { cause, opaque_types_jank },
)
}
/// Computes the region constraints and *new* opaque types registered when
/// proving a goal.
///
/// If an opaque was already constrained before proving this goal, then the
/// external constraints do not need to record that opaque, since if it is
/// further constrained by inference, that will be passed back in the var
/// values.
#[instrument(level = "trace", skip(self), ret)]
fn compute_external_query_constraints(
&self,
certainty: Certainty,
normalization_nested_goals: NestedNormalizationGoals<I>,
) -> ExternalConstraintsData<I> {
// We only return region constraints once the certainty is `Yes`. This
// is necessary as we may drop nested goals on ambiguity, which may result
// in unconstrained inference variables in the region constraints. It also
// prevents us from emitting duplicate region constraints, avoiding some
// unnecessary work. This slightly weakens the leak check in case it uses
// region constraints from an ambiguous nested goal. This is tested in both
// `tests/ui/higher-ranked/leak-check/leak-check-in-selection-5-ambig.rs` and
// `tests/ui/higher-ranked/leak-check/leak-check-in-selection-6-ambig-unify.rs`.
let region_constraints = if certainty == Certainty::Yes {
self.delegate.make_deduplicated_outlives_constraints()
} else {
Default::default()
};
// We only return *newly defined* opaque types from canonical queries.
//
// Constraints for any existing opaque types are already tracked by changes
// to the `var_values`.
let opaque_types = self
.delegate
.clone_opaque_types_added_since(self.initial_opaque_types_storage_num_entries);
ExternalConstraintsData { region_constraints, opaque_types, normalization_nested_goals }
}
/// After calling a canonical query, we apply the constraints returned
/// by the query using this function.
///
/// This happens in three steps:
/// - we instantiate the bound variables of the query response
/// - we unify the `var_values` of the response with the `original_values`
/// - we apply the `external_constraints` returned by the query, returning
/// the `normalization_nested_goals`
pub(super) fn instantiate_and_apply_query_response(
delegate: &D,
param_env: I::ParamEnv,
original_values: &[I::GenericArg],
response: CanonicalResponse<I>,
span: I::Span,
) -> (NestedNormalizationGoals<I>, Certainty) {
let instantiation = Self::compute_query_response_instantiation_values(
delegate,
&original_values,
&response,
span,
);
let Response { var_values, external_constraints, certainty } =
delegate.instantiate_canonical(response, instantiation);
Self::unify_query_var_values(delegate, param_env, &original_values, var_values, span);
let ExternalConstraintsData {
region_constraints,
opaque_types,
normalization_nested_goals,
} = &*external_constraints;
Self::register_region_constraints(delegate, region_constraints, span);
Self::register_new_opaque_types(delegate, opaque_types, span);
(normalization_nested_goals.clone(), certainty)
}
/// This returns the canonical variable values to instantiate the bound variables of
/// the canonical response. This depends on the `original_values` for the
/// bound variables.
fn compute_query_response_instantiation_values<T: ResponseT<I>>(
delegate: &D,
original_values: &[I::GenericArg],
response: &Canonical<I, T>,
span: I::Span,
) -> CanonicalVarValues<I> {
// FIXME: Longterm canonical queries should deal with all placeholders
// created inside of the query directly instead of returning them to the
// caller.
let prev_universe = delegate.universe();
let universes_created_in_query = response.max_universe.index();
for _ in 0..universes_created_in_query {
delegate.create_next_universe();
}
let var_values = response.value.var_values();
assert_eq!(original_values.len(), var_values.len());
// If the query did not make progress with constraining inference variables,
// we would normally create a new inference variables for bound existential variables
// only then unify this new inference variable with the inference variable from
// the input.
//
// We therefore instantiate the existential variable in the canonical response with the
// inference variable of the input right away, which is more performant.
let mut opt_values = IndexVec::from_elem_n(None, response.variables.len());
for (original_value, result_value) in
iter::zip(original_values, var_values.var_values.iter())
{
match result_value.kind() {
ty::GenericArgKind::Type(t) => {
// We disable the instantiation guess for inference variables
// and only use it for placeholders. We need to handle the
// `sub_root` of type inference variables which would make this
// more involved. They are also a lot rarer than region variables.
if let ty::Bound(debruijn, b) = t.kind()
&& !matches!(
response.variables.get(b.var().as_usize()).unwrap(),
CanonicalVarKind::Ty { .. }
)
{
assert_eq!(debruijn, ty::INNERMOST);
opt_values[b.var()] = Some(*original_value);
}
}
ty::GenericArgKind::Lifetime(r) => {
if let ty::ReBound(debruijn, br) = r.kind() {
assert_eq!(debruijn, ty::INNERMOST);
opt_values[br.var()] = Some(*original_value);
}
}
ty::GenericArgKind::Const(c) => {
if let ty::ConstKind::Bound(debruijn, bv) = c.kind() {
assert_eq!(debruijn, ty::INNERMOST);
opt_values[bv.var()] = Some(*original_value);
}
}
}
}
CanonicalVarValues::instantiate(delegate.cx(), response.variables, |var_values, kind| {
if kind.universe() != ty::UniverseIndex::ROOT {
// A variable from inside a binder of the query. While ideally these shouldn't
// exist at all (see the FIXME at the start of this method), we have to deal with
// them for now.
delegate.instantiate_canonical_var(kind, span, &var_values, |idx| {
prev_universe + idx.index()
})
} else if kind.is_existential() {
// As an optimization we sometimes avoid creating a new inference variable here.
//
// All new inference variables we create start out in the current universe of the caller.
// This is conceptually wrong as these inference variables would be able to name
// more placeholders then they should be able to. However the inference variables have
// to "come from somewhere", so by equating them with the original values of the caller
// later on, we pull them down into their correct universe again.
if let Some(v) = opt_values[ty::BoundVar::from_usize(var_values.len())] {
v
} else {
delegate.instantiate_canonical_var(kind, span, &var_values, |_| prev_universe)
}
} else {
// For placeholders which were already part of the input, we simply map this
// universal bound variable back the placeholder of the input.
original_values[kind.expect_placeholder_index()]
}
})
}
/// Unify the `original_values` with the `var_values` returned by the canonical query..
///
/// This assumes that this unification will always succeed. This is the case when
/// applying a query response right away. However, calling a canonical query, doing any
/// other kind of trait solving, and only then instantiating the result of the query
/// can cause the instantiation to fail. This is not supported and we ICE in this case.
///
/// We always structurally instantiate aliases. Relating aliases needs to be different
/// depending on whether the alias is *rigid* or not. We're only really able to tell
/// whether an alias is rigid by using the trait solver. When instantiating a response
/// from the solver we assume that the solver correctly handled aliases and therefore
/// always relate them structurally here.
#[instrument(level = "trace", skip(delegate))]
fn unify_query_var_values(
delegate: &D,
param_env: I::ParamEnv,
original_values: &[I::GenericArg],
var_values: CanonicalVarValues<I>,
span: I::Span,
) {
assert_eq!(original_values.len(), var_values.len());
for (&orig, response) in iter::zip(original_values, var_values.var_values.iter()) {
let goals =
delegate.eq_structurally_relating_aliases(param_env, orig, response, span).unwrap();
assert!(goals.is_empty());
}
}
fn register_region_constraints(
delegate: &D,
outlives: &[ty::OutlivesPredicate<I, I::GenericArg>],
span: I::Span,
) {
for &ty::OutlivesPredicate(lhs, rhs) in outlives {
match lhs.kind() {
ty::GenericArgKind::Lifetime(lhs) => delegate.sub_regions(rhs, lhs, span),
ty::GenericArgKind::Type(lhs) => delegate.register_ty_outlives(lhs, rhs, span),
ty::GenericArgKind::Const(_) => panic!("const outlives: {lhs:?}: {rhs:?}"),
}
}
}
fn register_new_opaque_types(
delegate: &D,
opaque_types: &[(ty::OpaqueTypeKey<I>, I::Ty)],
span: I::Span,
) {
for &(key, ty) in opaque_types {
let prev = delegate.register_hidden_type_in_storage(key, ty, span);
// We eagerly resolve inference variables when computing the query response.
// This can cause previously distinct opaque type keys to now be structurally equal.
//
// To handle this, we store any duplicate entries in a separate list to check them
// at the end of typeck/borrowck. We could alternatively eagerly equate the hidden
// types here. However, doing so is difficult as it may result in nested goals and
// any errors may make it harder to track the control flow for diagnostics.
if let Some(prev) = prev {
delegate.add_duplicate_opaque_type(key, prev, span);
}
}
}
}
/// Used by proof trees to be able to recompute intermediate actions while
/// evaluating a goal. The `var_values` not only include the bound variables
/// of the query input, but also contain all unconstrained inference vars
/// created while evaluating this goal.
pub(in crate::solve) fn make_canonical_state<D, T, I>(
delegate: &D,
var_values: &[I::GenericArg],
max_input_universe: ty::UniverseIndex,
data: T,
) -> inspect::CanonicalState<I, T>
where
D: SolverDelegate<Interner = I>,
I: Interner,
T: TypeFoldable<I>,
{
let var_values = CanonicalVarValues { var_values: delegate.cx().mk_args(var_values) };
let state = inspect::State { var_values, data };
let state = eager_resolve_vars(delegate, state);
Canonicalizer::canonicalize_response(delegate, max_input_universe, &mut vec![], state)
}
// FIXME: needs to be pub to be accessed by downstream
// `rustc_trait_selection::solve::inspect::analyse`.
pub fn instantiate_canonical_state<D, I, T: TypeFoldable<I>>(
delegate: &D,
span: I::Span,
param_env: I::ParamEnv,
orig_values: &mut Vec<I::GenericArg>,
state: inspect::CanonicalState<I, T>,
) -> T
where
D: SolverDelegate<Interner = I>,
I: Interner,
{
// In case any fresh inference variables have been created between `state`
// and the previous instantiation, extend `orig_values` for it.
orig_values.extend(
state.value.var_values.var_values.as_slice()[orig_values.len()..]
.iter()
.map(|&arg| delegate.fresh_var_for_kind_with_span(arg, span)),
);
let instantiation =
EvalCtxt::compute_query_response_instantiation_values(delegate, orig_values, &state, span);
let inspect::State { var_values, data } = delegate.instantiate_canonical(state, instantiation);
EvalCtxt::unify_query_var_values(delegate, param_env, orig_values, var_values, span);
data
}

212
compiler/rustc_next_trait_solver/src/solve/eval_ctxt/mod.rs
View file

@ -17,6 +17,10 @@ use rustc_type_ir::{
use tracing::{debug, instrument, trace};
use super::has_only_region_constraints;
use crate::canonical::{
canonicalize_goal, canonicalize_response, instantiate_and_apply_query_response,
response_no_constraints_raw,
};
use crate::coherence;
use crate::delegate::SolverDelegate;
use crate::placeholder::BoundVarReplacer;
@ -24,12 +28,11 @@ use crate::resolve::eager_resolve_vars;
use crate::solve::search_graph::SearchGraph;
use crate::solve::ty::may_use_unstable_feature;
use crate::solve::{
CanonicalInput, Certainty, FIXPOINT_STEP_LIMIT, Goal, GoalEvaluation, GoalSource,
GoalStalledOn, HasChanged, NestedNormalizationGoals, NoSolution, QueryInput, QueryResult,
inspect,
CanonicalInput, CanonicalResponse, Certainty, ExternalConstraintsData, FIXPOINT_STEP_LIMIT,
Goal, GoalEvaluation, GoalSource, GoalStalledOn, HasChanged, MaybeCause,
NestedNormalizationGoals, NoSolution, QueryInput, QueryResult, Response, inspect,
};
pub(super) mod canonical;
mod probe;
/// The kind of goal we're currently proving.
@ -464,8 +467,7 @@ where
let opaque_types = self.delegate.clone_opaque_types_lookup_table();
let (goal, opaque_types) = eager_resolve_vars(self.delegate, (goal, opaque_types));
let (orig_values, canonical_goal) =
Self::canonicalize_goal(self.delegate, goal, opaque_types);
let (orig_values, canonical_goal) = canonicalize_goal(self.delegate, goal, opaque_types);
let canonical_result = self.search_graph.evaluate_goal(
self.cx(),
canonical_goal,
@ -480,7 +482,7 @@ where
let has_changed =
if !has_only_region_constraints(response) { HasChanged::Yes } else { HasChanged::No };
let (normalization_nested_goals, certainty) = Self::instantiate_and_apply_query_response(
let (normalization_nested_goals, certainty) = instantiate_and_apply_query_response(
self.delegate,
goal.param_env,
&orig_values,
@ -1223,6 +1225,198 @@ where
vec![]
}
}
/// To return the constraints of a canonical query to the caller, we canonicalize:
///
/// - `var_values`: a map from bound variables in the canonical goal to
/// the values inferred while solving the instantiated goal.
/// - `external_constraints`: additional constraints which aren't expressible
/// using simple unification of inference variables.
///
/// This takes the `shallow_certainty` which represents whether we're confident
/// that the final result of the current goal only depends on the nested goals.
///
/// In case this is `Certainty::Maybe`, there may still be additional nested goals
/// or inference constraints required for this candidate to be hold. The candidate
/// always requires all already added constraints and nested goals.
#[instrument(level = "trace", skip(self), ret)]
pub(in crate::solve) fn evaluate_added_goals_and_make_canonical_response(
&mut self,
shallow_certainty: Certainty,
) -> QueryResult<I> {
self.inspect.make_canonical_response(shallow_certainty);
let goals_certainty = self.try_evaluate_added_goals()?;
assert_eq!(
self.tainted,
Ok(()),
"EvalCtxt is tainted -- nested goals may have been dropped in a \
previous call to `try_evaluate_added_goals!`"
);
// We only check for leaks from universes which were entered inside
// of the query.
self.delegate.leak_check(self.max_input_universe).map_err(|NoSolution| {
trace!("failed the leak check");
NoSolution
})?;
let (certainty, normalization_nested_goals) =
match (self.current_goal_kind, shallow_certainty) {
// When normalizing, we've replaced the expected term with an unconstrained
// inference variable. This means that we dropped information which could
// have been important. We handle this by instead returning the nested goals
// to the caller, where they are then handled. We only do so if we do not
// need to recompute the `NormalizesTo` goal afterwards to avoid repeatedly
// uplifting its nested goals. This is the case if the `shallow_certainty` is
// `Certainty::Yes`.
(CurrentGoalKind::NormalizesTo, Certainty::Yes) => {
let goals = std::mem::take(&mut self.nested_goals);
// As we return all ambiguous nested goals, we can ignore the certainty
// returned by `self.try_evaluate_added_goals()`.
if goals.is_empty() {
assert!(matches!(goals_certainty, Certainty::Yes));
}
(
Certainty::Yes,
NestedNormalizationGoals(
goals.into_iter().map(|(s, g, _)| (s, g)).collect(),
),
)
}
_ => {
let certainty = shallow_certainty.and(goals_certainty);
(certainty, NestedNormalizationGoals::empty())
}
};
if let Certainty::Maybe {
cause: cause @ MaybeCause::Overflow { keep_constraints: false, .. },
opaque_types_jank,
} = certainty
{
// If we have overflow, it's probable that we're substituting a type
// into itself infinitely and any partial substitutions in the query
// response are probably not useful anyways, so just return an empty
// query response.
//
// This may prevent us from potentially useful inference, e.g.
// 2 candidates, one ambiguous and one overflow, which both
// have the same inference constraints.
//
// Changing this to retain some constraints in the future
// won't be a breaking change, so this is good enough for now.
return Ok(self.make_ambiguous_response_no_constraints(cause, opaque_types_jank));
}
let external_constraints =
self.compute_external_query_constraints(certainty, normalization_nested_goals);
let (var_values, mut external_constraints) =
eager_resolve_vars(self.delegate, (self.var_values, external_constraints));
// Remove any trivial or duplicated region constraints once we've resolved regions
let mut unique = HashSet::default();
external_constraints.region_constraints.retain(|outlives| {
outlives.0.as_region().is_none_or(|re| re != outlives.1) && unique.insert(*outlives)
});
let canonical = canonicalize_response(
self.delegate,
self.max_input_universe,
Response {
var_values,
certainty,
external_constraints: self.cx().mk_external_constraints(external_constraints),
},
);
// HACK: We bail with overflow if the response would have too many non-region
// inference variables. This tends to only happen if we encounter a lot of
// ambiguous alias types which get replaced with fresh inference variables
// during generalization. This prevents hangs caused by an exponential blowup,
// see tests/ui/traits/next-solver/coherence-alias-hang.rs.
match self.current_goal_kind {
// We don't do so for `NormalizesTo` goals as we erased the expected term and
// bailing with overflow here would prevent us from detecting a type-mismatch,
// causing a coherence error in diesel, see #131969. We still bail with overflow
// when later returning from the parent AliasRelate goal.
CurrentGoalKind::NormalizesTo => {}
CurrentGoalKind::Misc | CurrentGoalKind::CoinductiveTrait => {
let num_non_region_vars = canonical
.variables
.iter()
.filter(|c| !c.is_region() && c.is_existential())
.count();
if num_non_region_vars > self.cx().recursion_limit() {
debug!(?num_non_region_vars, "too many inference variables -> overflow");
return Ok(self.make_ambiguous_response_no_constraints(
MaybeCause::Overflow {
suggest_increasing_limit: true,
keep_constraints: false,
},
OpaqueTypesJank::AllGood,
));
}
}
}
Ok(canonical)
}
/// Constructs a totally unconstrained, ambiguous response to a goal.
///
/// Take care when using this, since often it's useful to respond with
/// ambiguity but return constrained variables to guide inference.
pub(in crate::solve) fn make_ambiguous_response_no_constraints(
&self,
cause: MaybeCause,
opaque_types_jank: OpaqueTypesJank,
) -> CanonicalResponse<I> {
response_no_constraints_raw(
self.cx(),
self.max_input_universe,
self.variables,
Certainty::Maybe { cause, opaque_types_jank },
)
}
/// Computes the region constraints and *new* opaque types registered when
/// proving a goal.
///
/// If an opaque was already constrained before proving this goal, then the
/// external constraints do not need to record that opaque, since if it is
/// further constrained by inference, that will be passed back in the var
/// values.
#[instrument(level = "trace", skip(self), ret)]
fn compute_external_query_constraints(
&self,
certainty: Certainty,
normalization_nested_goals: NestedNormalizationGoals<I>,
) -> ExternalConstraintsData<I> {
// We only return region constraints once the certainty is `Yes`. This
// is necessary as we may drop nested goals on ambiguity, which may result
// in unconstrained inference variables in the region constraints. It also
// prevents us from emitting duplicate region constraints, avoiding some
// unnecessary work. This slightly weakens the leak check in case it uses
// region constraints from an ambiguous nested goal. This is tested in both
// `tests/ui/higher-ranked/leak-check/leak-check-in-selection-5-ambig.rs` and
// `tests/ui/higher-ranked/leak-check/leak-check-in-selection-6-ambig-unify.rs`.
let region_constraints = if certainty == Certainty::Yes {
self.delegate.make_deduplicated_outlives_constraints()
} else {
Default::default()
};
// We only return *newly defined* opaque types from canonical queries.
//
// Constraints for any existing opaque types are already tracked by changes
// to the `var_values`.
let opaque_types = self
.delegate
.clone_opaque_types_added_since(self.initial_opaque_types_storage_num_entries);
ExternalConstraintsData { region_constraints, opaque_types, normalization_nested_goals }
}
}
/// Eagerly replace aliases with inference variables, emitting `AliasRelate`
@ -1363,7 +1557,7 @@ pub(super) fn evaluate_root_goal_for_proof_tree<D: SolverDelegate<Interner = I>,
let opaque_types = delegate.clone_opaque_types_lookup_table();
let (goal, opaque_types) = eager_resolve_vars(delegate, (goal, opaque_types));
let (orig_values, canonical_goal) = EvalCtxt::canonicalize_goal(delegate, goal, opaque_types);
let (orig_values, canonical_goal) = canonicalize_goal(delegate, goal, opaque_types);
let (canonical_result, final_revision) =
delegate.cx().evaluate_root_goal_for_proof_tree_raw(canonical_goal);
@ -1380,7 +1574,7 @@ pub(super) fn evaluate_root_goal_for_proof_tree<D: SolverDelegate<Interner = I>,
Ok(response) => response,
};
let (normalization_nested_goals, _certainty) = EvalCtxt::instantiate_and_apply_query_response(
let (normalization_nested_goals, _certainty) = instantiate_and_apply_query_response(
delegate,
goal.param_env,
&proof_tree.orig_values,

2
compiler/rustc_next_trait_solver/src/solve/inspect/build.rs
View file

@ -10,8 +10,8 @@ use derive_where::derive_where;
use rustc_type_ir::inherent::*;
use rustc_type_ir::{self as ty, Interner};
use crate::canonical;
use crate::delegate::SolverDelegate;
use crate::solve::eval_ctxt::canonical;
use crate::solve::{Certainty, Goal, GoalSource, QueryResult, inspect};
/// We need to know whether to build a prove tree while evaluating. We

2
compiler/rustc_next_trait_solver/src/solve/inspect/mod.rs
View file

@ -2,5 +2,3 @@ pub use rustc_type_ir::solve::inspect::*;
mod build;
pub(in crate::solve) use build::*;
pub use crate::solve::eval_ctxt::canonical::instantiate_canonical_state;

19
compiler/rustc_next_trait_solver/src/solve/mod.rs
View file

@ -380,25 +380,6 @@ where
}
}
fn response_no_constraints_raw<I: Interner>(
cx: I,
max_universe: ty::UniverseIndex,
variables: I::CanonicalVarKinds,
certainty: Certainty,
) -> CanonicalResponse<I> {
ty::Canonical {
max_universe,
variables,
value: Response {
var_values: ty::CanonicalVarValues::make_identity(cx, variables),
// FIXME: maybe we should store the "no response" version in cx, like
// we do for cx.types and stuff.
external_constraints: cx.mk_external_constraints(ExternalConstraintsData::default()),
certainty,
},
}
}
/// The result of evaluating a goal.
pub struct GoalEvaluation<I: Interner> {
/// The goal we've evaluated. This is the input goal, but potentially with its

3
compiler/rustc_next_trait_solver/src/solve/search_graph.rs
View file

@ -6,6 +6,7 @@ use rustc_type_ir::search_graph::{self, PathKind};
use rustc_type_ir::solve::{CanonicalInput, Certainty, NoSolution, QueryResult};
use rustc_type_ir::{Interner, TypingMode};
use crate::canonical::response_no_constraints_raw;
use crate::delegate::SolverDelegate;
use crate::solve::{
EvalCtxt, FIXPOINT_STEP_LIMIT, has_no_inference_or_external_constraints, inspect,
@ -127,7 +128,7 @@ fn response_no_constraints<I: Interner>(
input: CanonicalInput<I>,
certainty: Certainty,
) -> QueryResult<I> {
Ok(super::response_no_constraints_raw(
Ok(response_no_constraints_raw(
cx,
input.canonical.max_universe,
input.canonical.variables,

5
compiler/rustc_session/src/config.rs
View file

@ -1509,6 +1509,11 @@ impl Options {
pub fn get_symbol_mangling_version(&self) -> SymbolManglingVersion {
self.cg.symbol_mangling_version.unwrap_or(SymbolManglingVersion::Legacy)
}
#[inline]
pub fn autodiff_enabled(&self) -> bool {
self.unstable_opts.autodiff.contains(&AutoDiff::Enable)
}
}
impl UnstableOptions {

7
compiler/rustc_session/src/session.rs
View file

@ -600,6 +600,13 @@ impl Session {
/// Calculates the flavor of LTO to use for this compilation.
pub fn lto(&self) -> config::Lto {
// Autodiff currently requires fat-lto to have access to the llvm-ir of all (indirectly) used functions and types.
// fat-lto is the easiest solution to this requirement, but quite expensive.
// FIXME(autodiff): Make autodiff also work with embed-bc instead of fat-lto.
if self.opts.autodiff_enabled() {
return config::Lto::Fat;
}
// If our target has codegen requirements ignore the command line
if self.target.requires_lto {
return config::Lto::Fat;

4
compiler/rustc_trait_selection/src/solve/inspect/analyse.rs
View file

@ -18,9 +18,9 @@ use rustc_middle::traits::ObligationCause;
use rustc_middle::traits::solve::{Certainty, Goal, GoalSource, NoSolution, QueryResult};
use rustc_middle::ty::{TyCtxt, VisitorResult, try_visit};
use rustc_middle::{bug, ty};
use rustc_next_trait_solver::canonical::instantiate_canonical_state;
use rustc_next_trait_solver::resolve::eager_resolve_vars;
use rustc_next_trait_solver::solve::inspect::{self, instantiate_canonical_state};
use rustc_next_trait_solver::solve::{MaybeCause, SolverDelegateEvalExt as _};
use rustc_next_trait_solver::solve::{MaybeCause, SolverDelegateEvalExt as _, inspect};
use rustc_span::Span;
use tracing::instrument;

7
compiler/rustc_windows_rc/src/lib.rs
View file

@ -35,8 +35,11 @@ pub fn compile_windows_resource_file(
resources_dir.push("resources");
fs::create_dir_all(&resources_dir).unwrap();
let resource_compiler =
find_resource_compiler(&env::var("CARGO_CFG_TARGET_ARCH").unwrap()).expect("found rc.exe");
let resource_compiler = if let Ok(path) = env::var("RUSTC_WINDOWS_RC") {
path.into()
} else {
find_resource_compiler(&env::var("CARGO_CFG_TARGET_ARCH").unwrap()).expect("found rc.exe")
};
let rc_path = resources_dir.join(file_stem.with_extension("rc"));

41
library/std/src/net/socket_addr.rs
View file

@ -6,7 +6,6 @@ mod tests;
pub use core::net::{SocketAddr, SocketAddrV4, SocketAddrV6};
use crate::net::{IpAddr, Ipv4Addr, Ipv6Addr};
use crate::sys::net::LookupHost;
use crate::{io, iter, option, slice, vec};
/// A trait for objects which can be converted or resolved to one or more
@ -188,15 +187,9 @@ impl ToSocketAddrs for (Ipv6Addr, u16) {
}
}
fn resolve_socket_addr(lh: LookupHost) -> io::Result<vec::IntoIter<SocketAddr>> {
let p = lh.port();
let v: Vec<_> = lh
.map(|mut a| {
a.set_port(p);
a
})
.collect();
Ok(v.into_iter())
fn lookup_host(host: &str, port: u16) -> io::Result<vec::IntoIter<SocketAddr>> {
let addrs = crate::sys::net::lookup_host(host, port)?;
Ok(Vec::from_iter(addrs).into_iter())
}
#[stable(feature = "rust1", since = "1.0.0")]
@ -205,17 +198,14 @@ impl ToSocketAddrs for (&str, u16) {
fn to_socket_addrs(&self) -> io::Result<vec::IntoIter<SocketAddr>> {
let (host, port) = *self;
// try to parse the host as a regular IP address first
if let Ok(addr) = host.parse::<Ipv4Addr>() {
let addr = SocketAddrV4::new(addr, port);
return Ok(vec![SocketAddr::V4(addr)].into_iter());
}
if let Ok(addr) = host.parse::<Ipv6Addr>() {
let addr = SocketAddrV6::new(addr, port, 0, 0);
return Ok(vec![SocketAddr::V6(addr)].into_iter());
// Try to parse the host as a regular IP address first
if let Ok(addr) = host.parse::<IpAddr>() {
let addr = SocketAddr::new(addr, port);
return Ok(vec![addr].into_iter());
}
resolve_socket_addr((host, port).try_into()?)
// Otherwise, make the system look it up.
lookup_host(host, port)
}
}
@ -232,12 +222,21 @@ impl ToSocketAddrs for (String, u16) {
impl ToSocketAddrs for str {
type Iter = vec::IntoIter<SocketAddr>;
fn to_socket_addrs(&self) -> io::Result<vec::IntoIter<SocketAddr>> {
// try to parse as a regular SocketAddr first
// Try to parse as a regular SocketAddr first
if let Ok(addr) = self.parse() {
return Ok(vec![addr].into_iter());
}
resolve_socket_addr(self.try_into()?)
// Otherwise, split the string by ':' and convert the second part to u16...
let Some((host, port_str)) = self.rsplit_once(':') else {
return Err(io::const_error!(io::ErrorKind::InvalidInput, "invalid socket address"));
};
let Ok(port) = port_str.parse::<u16>() else {
return Err(io::const_error!(io::ErrorKind::InvalidInput, "invalid port value"));
};
// ... and make the system look up the host.
lookup_host(host, port)
}
}

9
library/std/src/sys/fs/wasi.rs
View file

@ -848,7 +848,14 @@ fn remove_dir_all_recursive(parent: &WasiFd, path: &Path) -> io::Result<()> {
// Iterate over all the entries in this directory, and travel recursively if
// necessary
for entry in ReadDir::new(fd, dummy_root) {
//
// Note that all directory entries for this directory are read first before
// any removal is done. This works around the fact that the WASIp1 API for
// reading directories is not well-designed for handling mutations between
// invocations of reading a directory. By reading all the entries at once
// this ensures that, at least without concurrent modifications, it should
// be possible to delete everything.
for entry in ReadDir::new(fd, dummy_root).collect::<Vec<_>>() {
let entry = entry?;
let path = crate::str::from_utf8(&entry.name).map_err(|_| {
io::const_error!(io::ErrorKind::Uncategorized, "invalid utf-8 file name found")

29
library/std/src/sys/net/connection/sgx.rs
View file

@ -499,16 +499,6 @@ impl fmt::Display for NonIpSockAddr {
pub struct LookupHost(!);
impl LookupHost {
fn new(host: String) -> io::Result<LookupHost> {
Err(io::Error::new(io::ErrorKind::Uncategorized, NonIpSockAddr { host }))
}
pub fn port(&self) -> u16 {
self.0
}
}
impl Iterator for LookupHost {
type Item = SocketAddr;
fn next(&mut self) -> Option<SocketAddr> {
@ -516,18 +506,9 @@ impl Iterator for LookupHost {
}
}
impl TryFrom<&str> for LookupHost {
type Error = io::Error;
fn try_from(v: &str) -> io::Result<LookupHost> {
LookupHost::new(v.to_owned())
}
}
impl<'a> TryFrom<(&'a str, u16)> for LookupHost {
type Error = io::Error;
fn try_from((host, port): (&'a str, u16)) -> io::Result<LookupHost> {
LookupHost::new(format!("{host}:{port}"))
}
pub fn lookup_host(host: &str, port: u16) -> io::Result<LookupHost> {
Err(io::Error::new(
io::ErrorKind::Uncategorized,
NonIpSockAddr { host: format!("{host}:{port}") },
))
}

58
library/std/src/sys/net/connection/socket/mod.rs
View file

@ -258,7 +258,7 @@ fn to_ipv6mr_interface(value: u32) -> crate::ffi::c_uint {
}
////////////////////////////////////////////////////////////////////////////////
// get_host_addresses
// lookup_host
////////////////////////////////////////////////////////////////////////////////
pub struct LookupHost {
@ -267,12 +267,6 @@ pub struct LookupHost {
port: u16,
}
impl LookupHost {
pub fn port(&self) -> u16 {
self.port
}
}
impl Iterator for LookupHost {
type Item = SocketAddr;
fn next(&mut self) -> Option<SocketAddr> {
@ -281,7 +275,10 @@ impl Iterator for LookupHost {
let cur = self.cur.as_ref()?;
self.cur = cur.ai_next;
match socket_addr_from_c(cur.ai_addr.cast(), cur.ai_addrlen as usize) {
Ok(addr) => return Some(addr),
Ok(mut addr) => {
addr.set_port(self.port);
return Some(addr);
}
Err(_) => continue,
}
}
@ -298,42 +295,17 @@ impl Drop for LookupHost {
}
}
impl TryFrom<&str> for LookupHost {
type Error = io::Error;
fn try_from(s: &str) -> io::Result<LookupHost> {
macro_rules! try_opt {
($e:expr, $msg:expr) => {
match $e {
Some(r) => r,
None => return Err(io::const_error!(io::ErrorKind::InvalidInput, $msg)),
}
};
pub fn lookup_host(host: &str, port: u16) -> io::Result<LookupHost> {
init();
run_with_cstr(host.as_bytes(), &|c_host| {
let mut hints: c::addrinfo = unsafe { mem::zeroed() };
hints.ai_socktype = c::SOCK_STREAM;
let mut res = ptr::null_mut();
unsafe {
cvt_gai(c::getaddrinfo(c_host.as_ptr(), ptr::null(), &hints, &mut res))
.map(|_| LookupHost { original: res, cur: res, port })
}
// split the string by ':' and convert the second part to u16
let (host, port_str) = try_opt!(s.rsplit_once(':'), "invalid socket address");
let port: u16 = try_opt!(port_str.parse().ok(), "invalid port value");
(host, port).try_into()
}
}
impl<'a> TryFrom<(&'a str, u16)> for LookupHost {
type Error = io::Error;
fn try_from((host, port): (&'a str, u16)) -> io::Result<LookupHost> {
init();
run_with_cstr(host.as_bytes(), &|c_host| {
let mut hints: c::addrinfo = unsafe { mem::zeroed() };
hints.ai_socktype = c::SOCK_STREAM;
let mut res = ptr::null_mut();
unsafe {
cvt_gai(c::getaddrinfo(c_host.as_ptr(), ptr::null(), &hints, &mut res))
.map(|_| LookupHost { original: res, cur: res, port })
}
})
}
})
}
////////////////////////////////////////////////////////////////////////////////

2
library/std/src/sys/net/connection/socket/tests.rs
View file

@ -4,7 +4,7 @@ use crate::collections::HashMap;
#[test]
fn no_lookup_host_duplicates() {
let mut addrs = HashMap::new();
let lh = match LookupHost::try_from(("localhost", 0)) {
let lh = match lookup_host("localhost", 0) {
Ok(lh) => lh,
Err(e) => panic!("couldn't resolve `localhost`: {e}"),
};

22
library/std/src/sys/net/connection/uefi/mod.rs
View file

@ -333,12 +333,6 @@ impl fmt::Debug for UdpSocket {
pub struct LookupHost(!);
impl LookupHost {
pub fn port(&self) -> u16 {
self.0
}
}
impl Iterator for LookupHost {
type Item = SocketAddr;
fn next(&mut self) -> Option<SocketAddr> {
@ -346,18 +340,6 @@ impl Iterator for LookupHost {
}
}
impl TryFrom<&str> for LookupHost {
type Error = io::Error;
fn try_from(_v: &str) -> io::Result<LookupHost> {
unsupported()
}
}
impl<'a> TryFrom<(&'a str, u16)> for LookupHost {
type Error = io::Error;
fn try_from(_v: (&'a str, u16)) -> io::Result<LookupHost> {
unsupported()
}
pub fn lookup_host(_host: &str, _port: u16) -> io::Result<LookupHost> {
unsupported()
}

22
library/std/src/sys/net/connection/unsupported.rs
View file

@ -304,12 +304,6 @@ impl fmt::Debug for UdpSocket {
pub struct LookupHost(!);
impl LookupHost {
pub fn port(&self) -> u16 {
self.0
}
}
impl Iterator for LookupHost {
type Item = SocketAddr;
fn next(&mut self) -> Option<SocketAddr> {
@ -317,18 +311,6 @@ impl Iterator for LookupHost {
}
}
impl TryFrom<&str> for LookupHost {
type Error = io::Error;
fn try_from(_v: &str) -> io::Result<LookupHost> {
unsupported()
}
}
impl<'a> TryFrom<(&'a str, u16)> for LookupHost {
type Error = io::Error;
fn try_from(_v: (&'a str, u16)) -> io::Result<LookupHost> {
unsupported()
}
pub fn lookup_host(_host: &str, _port: u16) -> io::Result<LookupHost> {
unsupported()
}

22
library/std/src/sys/net/connection/wasip1.rs
View file

@ -477,12 +477,6 @@ impl fmt::Debug for UdpSocket {
pub struct LookupHost(!);
impl LookupHost {
pub fn port(&self) -> u16 {
self.0
}
}
impl Iterator for LookupHost {
type Item = SocketAddr;
fn next(&mut self) -> Option<SocketAddr> {
@ -490,18 +484,6 @@ impl Iterator for LookupHost {
}
}
impl<'a> TryFrom<&'a str> for LookupHost {
type Error = io::Error;
fn try_from(_v: &'a str) -> io::Result<LookupHost> {
unsupported()
}
}
impl<'a> TryFrom<(&'a str, u16)> for LookupHost {
type Error = io::Error;
fn try_from(_v: (&'a str, u16)) -> io::Result<LookupHost> {
unsupported()
}
pub fn lookup_host(_host: &str, _port: u16) -> io::Result<LookupHost> {
unsupported()
}

45
library/std/src/sys/net/connection/xous/dns.rs
View file

@ -1,15 +1,8 @@
use core::convert::{TryFrom, TryInto};
use crate::io;
use crate::net::{Ipv4Addr, SocketAddr, SocketAddrV4, SocketAddrV6};
use crate::os::xous::ffi::lend_mut;
use crate::os::xous::services::{DnsLendMut, dns_server};
pub struct DnsError {
#[allow(dead_code)]
pub code: u8,
}
#[repr(C, align(4096))]
struct LookupHostQuery([u8; 4096]);
@ -20,12 +13,6 @@ pub struct LookupHost {
count: usize,
}
impl LookupHost {
pub fn port(&self) -> u16 {
self.port
}
}
impl Iterator for LookupHost {
type Item = SocketAddr;
fn next(&mut self) -> Option<SocketAddr> {
@ -72,7 +59,7 @@ impl Iterator for LookupHost {
}
}
pub fn lookup(query: &str, port: u16) -> Result<LookupHost, DnsError> {
pub fn lookup_host(query: &str, port: u16) -> io::Result<LookupHost> {
let mut result = LookupHost { data: LookupHostQuery([0u8; 4096]), offset: 0, count: 0, port };
// Copy the query into the message that gets sent to the DNS server
@ -89,7 +76,7 @@ pub fn lookup(query: &str, port: u16) -> Result<LookupHost, DnsError> {
)
.unwrap();
if result.data.0[0] != 0 {
return Err(DnsError { code: result.data.0[1] });
return Err(io::const_error!(io::ErrorKind::InvalidInput, "DNS failure"));
}
assert_eq!(result.offset, 0);
result.count = result.data.0[1] as usize;
@ -98,31 +85,3 @@ pub fn lookup(query: &str, port: u16) -> Result<LookupHost, DnsError> {
result.offset = 2;
Ok(result)
}
impl TryFrom<&str> for LookupHost {
type Error = io::Error;
fn try_from(s: &str) -> io::Result<LookupHost> {
macro_rules! try_opt {
($e:expr, $msg:expr) => {
match $e {
Some(r) => r,
None => return Err(io::const_error!(io::ErrorKind::InvalidInput, &$msg)),
}
};
}
// split the string by ':' and convert the second part to u16
let (host, port_str) = try_opt!(s.rsplit_once(':'), "invalid socket address");
let port: u16 = try_opt!(port_str.parse().ok(), "invalid port value");
(host, port).try_into()
}
}
impl TryFrom<(&str, u16)> for LookupHost {
type Error = io::Error;
fn try_from(v: (&str, u16)) -> io::Result<LookupHost> {
lookup(v.0, v.1).map_err(|_e| io::const_error!(io::ErrorKind::InvalidInput, "DNS failure"))
}
}

2
library/std/src/sys/net/connection/xous/mod.rs
View file

@ -45,4 +45,4 @@ pub struct GetAddress {
raw: [u8; 4096],
}
pub use dns::LookupHost;
pub use dns::lookup_host;

16
library/stdarch/.cirrus.yml
View file

@ -1,16 +0,0 @@
task:
name: x86_64-unknown-freebsd
freebsd_instance:
image_family: freebsd-13-4
env:
# FIXME(freebsd): FreeBSD has a segfault when `RUST_BACKTRACE` is set
# https://github.com/rust-lang/rust/issues/132185
RUST_BACKTRACE: "0"
setup_script:
- curl https://sh.rustup.rs -sSf --output rustup.sh
- sh rustup.sh --default-toolchain nightly -y
- . $HOME/.cargo/env
- rustup default nightly
test_script:
- . $HOME/.cargo/env
- cargo build --all

10
library/stdarch/.github/workflows/main.yml vendored
View file

@ -117,6 +117,8 @@ jobs:
os: windows-2025
- tuple: aarch64-pc-windows-msvc
os: windows-11-arm
- tuple: arm64ec-pc-windows-msvc
os: windows-11-arm
- tuple: x86_64-pc-windows-gnu
os: windows-2025
# - tuple: i686-pc-windows-gnu
@ -207,14 +209,6 @@ jobs:
rustup update nightly --no-self-update
rustup default nightly
shell: bash
if: matrix.target.os != 'windows-11-arm'
- name: Install Rust for `windows-11-arm` runners
# The arm runners don't have Rust pre-installed (https://github.com/actions/partner-runner-images/issues/77)
run: |
curl https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
shell: bash
if: matrix.target.os == 'windows-11-arm'
- run: rustup target add ${{ matrix.target.tuple }}
shell: bash

217
library/stdarch/Cargo.lock
View file

@ -13,9 +13,9 @@ dependencies = [
[[package]]
name = "anstream"
version = "0.6.19"
version = "0.6.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "301af1932e46185686725e0fad2f8f2aa7da69dd70bf6ecc44d6b703844a3933"
checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192"
dependencies = [
"anstyle",
"anstyle-parse",
@ -43,29 +43,29 @@ dependencies = [
[[package]]
name = "anstyle-query"
version = "1.1.3"
version = "1.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c8bdeb6047d8983be085bab0ba1472e6dc604e7041dbf6fcd5e71523014fae9"
checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2"
dependencies = [
"windows-sys",
"windows-sys 0.60.2",
]
[[package]]
name = "anstyle-wincon"
version = "3.0.9"
version = "3.0.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "403f75924867bb1033c59fbf0797484329750cfbe3c4325cd33127941fabc882"
checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys",
"windows-sys 0.60.2",
]
[[package]]
name = "anyhow"
version = "1.0.98"
version = "1.0.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100"
[[package]]
name = "assert-instr-macro"
@ -84,30 +84,31 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "bitflags"
version = "2.9.1"
version = "2.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967"
checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394"
[[package]]
name = "cc"
version = "1.2.31"
version = "1.2.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3a42d84bb6b69d3a8b3eaacf0d88f179e1929695e1ad012b6cf64d9caaa5fd2"
checksum = "5252b3d2648e5eedbc1a6f501e3c795e07025c1e93bbf8bbdd6eef7f447a6d54"
dependencies = [
"find-msvc-tools",
"shlex",
]
[[package]]
name = "cfg-if"
version = "1.0.1"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268"
checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9"
[[package]]
name = "clap"
version = "4.5.42"
version = "4.5.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed87a9d530bb41a67537289bafcac159cb3ee28460e0a4571123d2a778a6a882"
checksum = "7eac00902d9d136acd712710d71823fb8ac8004ca445a89e73a41d45aa712931"
dependencies = [
"clap_builder",
"clap_derive",
@ -115,9 +116,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.42"
version = "4.5.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64f4f3f3c77c94aff3c7e9aac9a2ca1974a5adf392a8bb751e827d6d127ab966"
checksum = "2ad9bbf750e73b5884fb8a211a9424a1906c1e156724260fdae972f31d70e1d6"
dependencies = [
"anstream",
"anstyle",
@ -127,9 +128,9 @@ dependencies = [
[[package]]
name = "clap_derive"
version = "4.5.41"
version = "4.5.47"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef4f52386a59ca4c860f7393bcf8abd8dfd91ecccc0f774635ff68e92eeef491"
checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c"
dependencies = [
"heck",
"proc-macro2",
@ -258,6 +259,12 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "find-msvc-tools"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fd99930f64d146689264c637b5af2f0233a933bef0d8570e2526bf9e083192d"
[[package]]
name = "fnv"
version = "1.0.7"
@ -283,9 +290,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
[[package]]
name = "hashbrown"
version = "0.15.4"
version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
[[package]]
name = "heck"
@ -323,12 +330,12 @@ dependencies = [
[[package]]
name = "indexmap"
version = "2.10.0"
version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661"
checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9"
dependencies = [
"equivalent",
"hashbrown 0.15.4",
"hashbrown 0.15.5",
]
[[package]]
@ -353,7 +360,7 @@ checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9"
dependencies = [
"hermit-abi",
"libc",
"windows-sys",
"windows-sys 0.59.0",
]
[[package]]
@ -379,9 +386,9 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
[[package]]
name = "libc"
version = "0.2.174"
version = "0.2.175"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776"
checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543"
[[package]]
name = "linked-hash-map"
@ -391,9 +398,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "log"
version = "0.4.27"
version = "0.4.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94"
checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432"
[[package]]
name = "memchr"
@ -428,9 +435,9 @@ dependencies = [
[[package]]
name = "proc-macro2"
version = "1.0.95"
version = "1.0.101"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778"
checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de"
dependencies = [
"unicode-ident",
]
@ -497,9 +504,9 @@ dependencies = [
[[package]]
name = "rayon"
version = "1.10.0"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
dependencies = [
"either",
"rayon-core",
@ -507,9 +514,9 @@ dependencies = [
[[package]]
name = "rayon-core"
version = "1.12.1"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
dependencies = [
"crossbeam-deque",
"crossbeam-utils",
@ -517,9 +524,9 @@ dependencies = [
[[package]]
name = "regex"
version = "1.11.1"
version = "1.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191"
checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912"
dependencies = [
"aho-corasick",
"memchr",
@ -529,9 +536,9 @@ dependencies = [
[[package]]
name = "regex-automata"
version = "0.4.9"
version = "0.4.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908"
checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6"
dependencies = [
"aho-corasick",
"memchr",
@ -540,9 +547,9 @@ dependencies = [
[[package]]
name = "regex-syntax"
version = "0.8.5"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001"
[[package]]
name = "rustc-demangle"
@ -593,9 +600,9 @@ dependencies = [
[[package]]
name = "serde_json"
version = "1.0.142"
version = "1.0.143"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "030fedb782600dcbd6f02d479bf0d817ac3bb40d644745b769d6a96bc3afc5a7"
checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a"
dependencies = [
"itoa",
"memchr",
@ -715,9 +722,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]]
name = "syn"
version = "2.0.104"
version = "2.0.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40"
checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6"
dependencies = [
"proc-macro2",
"quote",
@ -774,7 +781,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "161296c618fa2d63f6ed5fffd1112937e803cb9ec71b32b01a76321555660917"
dependencies = [
"bitflags",
"indexmap 2.10.0",
"indexmap 2.11.0",
"semver",
]
@ -791,20 +798,35 @@ dependencies = [
[[package]]
name = "winapi-util"
version = "0.1.9"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
checksum = "0978bf7171b3d90bac376700cb56d606feb40f251a475a5d6634613564460b22"
dependencies = [
"windows-sys",
"windows-sys 0.60.2",
]
[[package]]
name = "windows-link"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a"
[[package]]
name = "windows-sys"
version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.60.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
dependencies = [
"windows-targets 0.53.3",
]
[[package]]
@ -813,14 +835,31 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_i686_gnullvm 0.52.6",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.53.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91"
dependencies = [
"windows-link",
"windows_aarch64_gnullvm 0.53.0",
"windows_aarch64_msvc 0.53.0",
"windows_i686_gnu 0.53.0",
"windows_i686_gnullvm 0.53.0",
"windows_i686_msvc 0.53.0",
"windows_x86_64_gnu 0.53.0",
"windows_x86_64_gnullvm 0.53.0",
"windows_x86_64_msvc 0.53.0",
]
[[package]]
@ -829,48 +868,96 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_aarch64_msvc"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
[[package]]
name = "windows_i686_gnu"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3"
[[package]]
name = "windows_i686_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_gnullvm"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_i686_msvc"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnu"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "windows_x86_64_msvc"
version = "0.53.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
[[package]]
name = "yaml-rust"
version = "0.4.5"
@ -882,18 +969,18 @@ dependencies = [
[[package]]
name = "zerocopy"
version = "0.8.26"
version = "0.8.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f"
checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c"
dependencies = [
"zerocopy-derive",
]
[[package]]
name = "zerocopy-derive"
version = "0.8.26"
version = "0.8.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181"
checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831"
dependencies = [
"proc-macro2",
"quote",

4
library/stdarch/ci/docker/loongarch64-unknown-linux-gnu/Dockerfile
View file

@ -2,11 +2,11 @@ FROM ubuntu:25.10
RUN apt-get update && \
apt-get install -y --no-install-recommends \
gcc libc6-dev qemu-user-static ca-certificates \
gcc libc6-dev qemu-user ca-certificates \
gcc-loongarch64-linux-gnu libc6-dev-loong64-cross
ENV CARGO_TARGET_LOONGARCH64_UNKNOWN_LINUX_GNU_LINKER=loongarch64-linux-gnu-gcc \
CARGO_TARGET_LOONGARCH64_UNKNOWN_LINUX_GNU_RUNNER="qemu-loongarch64-static -cpu max -L /usr/loongarch64-linux-gnu" \
CARGO_TARGET_LOONGARCH64_UNKNOWN_LINUX_GNU_RUNNER="qemu-loongarch64 -cpu max -L /usr/loongarch64-linux-gnu" \
OBJDUMP=loongarch64-linux-gnu-objdump \
STDARCH_TEST_SKIP_FEATURE=frecipe

1051
library/stdarch/crates/core_arch/src/aarch64/neon/generated.rs
View file

File diff suppressed because it is too large Load diff

9019
library/stdarch/crates/core_arch/src/arm_shared/neon/generated.rs
View file

File diff suppressed because it is too large Load diff

8
library/stdarch/crates/core_arch/src/arm_shared/neon/mod.rs
View file

@ -5503,8 +5503,12 @@ mod tests {
test_vcombine!(test_vcombine_s16 => vcombine_s16([3_i16, -4, 5, -6], [13_i16, -14, 15, -16]));
test_vcombine!(test_vcombine_u16 => vcombine_u16([3_u16, 4, 5, 6], [13_u16, 14, 15, 16]));
test_vcombine!(test_vcombine_p16 => vcombine_p16([3_u16, 4, 5, 6], [13_u16, 14, 15, 16]));
test_vcombine!(test_vcombine_f16 => vcombine_f16([3_f16, 4., 5., 6.],
[13_f16, 14., 15., 16.]));
#[cfg(not(target_arch = "arm64ec"))]
mod fp16 {
use super::*;
test_vcombine!(test_vcombine_f16 => vcombine_f16([3_f16, 4., 5., 6.],
[13_f16, 14., 15., 16.]));
}
test_vcombine!(test_vcombine_s32 => vcombine_s32([3_i32, -4], [13_i32, -14]));
test_vcombine!(test_vcombine_u32 => vcombine_u32([3_u32, 4], [13_u32, 14]));

3
library/stdarch/crates/core_arch/src/lib.rs
View file

@ -33,7 +33,8 @@
x86_amx_intrinsics,
f16,
aarch64_unstable_target_feature,
bigint_helper_methods
bigint_helper_methods,
funnel_shifts
)]
#![cfg_attr(test, feature(test, abi_vectorcall, stdarch_internal))]
#![deny(clippy::missing_inline_in_public_items)]

7
library/stdarch/crates/core_arch/src/loongarch32/mod.rs
View file

@ -17,9 +17,10 @@ unsafe extern "unadjusted" {
/// Generates the cache operation instruction
#[inline]
#[unstable(feature = "stdarch_loongarch", issue = "117427")]
pub unsafe fn cacop<const IMM12: i32>(a: i32, b: i32) {
static_assert_simm_bits!(IMM12, 12);
__cacop(a, b, IMM12);
pub unsafe fn cacop<const IMM5: i32, const IMM_S12: i32>(b: i32) {
static_assert_uimm_bits!(IMM5, 5);
static_assert_simm_bits!(IMM_S12, 12);
__cacop(IMM5, b, IMM_S12);
}
/// Reads the CSR

17
library/stdarch/crates/core_arch/src/loongarch64/mod.rs
View file

@ -64,9 +64,10 @@ pub fn crcc_w_d_w(a: i64, b: i32) -> i32 {
/// Generates the cache operation instruction
#[inline]
#[unstable(feature = "stdarch_loongarch", issue = "117427")]
pub unsafe fn cacop<const IMM12: i64>(a: i64, b: i64) {
static_assert_simm_bits!(IMM12, 12);
__cacop(a, b, IMM12);
pub unsafe fn cacop<const IMM5: i64, const IMM_S12: i64>(b: i64) {
static_assert_uimm_bits!(IMM5, 5);
static_assert_simm_bits!(IMM_S12, 12);
__cacop(IMM5, b, IMM_S12);
}
/// Reads the CSR
@ -125,14 +126,16 @@ pub unsafe fn asrtgt(a: i64, b: i64) {
#[inline]
#[rustc_legacy_const_generics(1)]
#[unstable(feature = "stdarch_loongarch", issue = "117427")]
pub unsafe fn lddir<const B: i64>(a: i64) -> i64 {
__lddir(a, B)
pub unsafe fn lddir<const IMM8: i64>(a: i64) -> i64 {
static_assert_uimm_bits!(IMM8, 8);
__lddir(a, IMM8)
}
/// Loads the page table entry
#[inline]
#[rustc_legacy_const_generics(1)]
#[unstable(feature = "stdarch_loongarch", issue = "117427")]
pub unsafe fn ldpte<const B: i64>(a: i64) {
__ldpte(a, B)
pub unsafe fn ldpte<const IMM8: i64>(a: i64) {
static_assert_uimm_bits!(IMM8, 8);
__ldpte(a, IMM8)
}

12
library/stdarch/crates/core_arch/src/loongarch_shared/mod.rs
View file

@ -131,17 +131,17 @@ pub fn ibar<const IMM15: i32>() {
/// Moves data from a GPR to the FCSR
#[inline]
#[unstable(feature = "stdarch_loongarch", issue = "117427")]
pub unsafe fn movgr2fcsr<const IMM5: i32>(a: i32) {
static_assert_uimm_bits!(IMM5, 5);
__movgr2fcsr(IMM5, a);
pub unsafe fn movgr2fcsr<const IMM2: i32>(a: i32) {
static_assert_uimm_bits!(IMM2, 2);
__movgr2fcsr(IMM2, a);
}
/// Moves data from a FCSR to the GPR
#[inline]
#[unstable(feature = "stdarch_loongarch", issue = "117427")]
pub fn movfcsr2gr<const IMM5: i32>() -> i32 {
static_assert_uimm_bits!(IMM5, 5);
unsafe { __movfcsr2gr(IMM5) }
pub fn movfcsr2gr<const IMM2: i32>() -> i32 {
static_assert_uimm_bits!(IMM2, 2);
unsafe { __movfcsr2gr(IMM2) }
}
/// Reads the 8-bit IO-CSR

21
library/stdarch/crates/core_arch/src/riscv64/mod.rs
View file

@ -20,7 +20,12 @@ pub use zk::*;
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hlv_wu(src: *const u32) -> u32 {
let value: u32;
asm!(".insn i 0x73, 0x4, {}, {}, 0x681", out(reg) value, in(reg) src, options(readonly, nostack));
asm!(
".insn i 0x73, 0x4, {}, {}, 0x681",
lateout(reg) value,
in(reg) src,
options(readonly, nostack, preserves_flags)
);
value
}
@ -38,7 +43,12 @@ pub unsafe fn hlv_wu(src: *const u32) -> u32 {
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hlv_d(src: *const i64) -> i64 {
let value: i64;
asm!(".insn i 0x73, 0x4, {}, {}, 0x6C0", out(reg) value, in(reg) src, options(readonly, nostack));
asm!(
".insn i 0x73, 0x4, {}, {}, 0x6C0",
lateout(reg) value,
in(reg) src,
options(readonly, nostack, preserves_flags)
);
value
}
@ -53,5 +63,10 @@ pub unsafe fn hlv_d(src: *const i64) -> i64 {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hsv_d(dst: *mut i64, src: i64) {
asm!(".insn r 0x73, 0x4, 0x37, x0, {}, {}", in(reg) dst, in(reg) src, options(nostack));
asm!(
".insn r 0x73, 0x4, 0x37, x0, {}, {}",
in(reg) dst,
in(reg) src,
options(nostack, preserves_flags)
);
}

2
library/stdarch/crates/core_arch/src/riscv64/zk.rs
View file

@ -176,7 +176,7 @@ pub fn aes64ks2(rs1: u64, rs2: u64) -> u64 {
/// Version: v1.0.1
///
/// Section: 3.9
#[target_feature(enable = "zkne", enable = "zknd")]
#[target_feature(enable = "zknd")]
#[cfg_attr(test, assert_instr(aes64im))]
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]

245
library/stdarch/crates/core_arch/src/riscv_shared/mod.rs
View file

@ -44,7 +44,12 @@ use crate::arch::asm;
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub fn pause() {
unsafe { asm!(".insn i 0x0F, 0, x0, x0, 0x010", options(nomem, nostack)) }
unsafe {
asm!(
".insn i 0x0F, 0, x0, x0, 0x010",
options(nomem, nostack, preserves_flags)
);
}
}
/// Generates the `NOP` instruction
@ -54,7 +59,9 @@ pub fn pause() {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub fn nop() {
unsafe { asm!("nop", options(nomem, nostack)) }
unsafe {
asm!("nop", options(nomem, nostack, preserves_flags));
}
}
/// Generates the `WFI` instruction
@ -65,7 +72,7 @@ pub fn nop() {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn wfi() {
asm!("wfi", options(nomem, nostack))
asm!("wfi", options(nomem, nostack, preserves_flags));
}
/// Generates the `FENCE.I` instruction
@ -78,7 +85,7 @@ pub unsafe fn wfi() {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn fence_i() {
asm!("fence.i", options(nostack))
asm!("fence.i", options(nostack, preserves_flags));
}
/// Supervisor memory management fence for given virtual address and address space
@ -92,7 +99,7 @@ pub unsafe fn fence_i() {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn sfence_vma(vaddr: usize, asid: usize) {
asm!("sfence.vma {}, {}", in(reg) vaddr, in(reg) asid, options(nostack))
asm!("sfence.vma {}, {}", in(reg) vaddr, in(reg) asid, options(nostack, preserves_flags));
}
/// Supervisor memory management fence for given virtual address
@ -104,7 +111,7 @@ pub unsafe fn sfence_vma(vaddr: usize, asid: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn sfence_vma_vaddr(vaddr: usize) {
asm!("sfence.vma {}, x0", in(reg) vaddr, options(nostack))
asm!("sfence.vma {}, x0", in(reg) vaddr, options(nostack, preserves_flags));
}
/// Supervisor memory management fence for given address space
@ -118,7 +125,7 @@ pub unsafe fn sfence_vma_vaddr(vaddr: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn sfence_vma_asid(asid: usize) {
asm!("sfence.vma x0, {}", in(reg) asid, options(nostack))
asm!("sfence.vma x0, {}", in(reg) asid, options(nostack, preserves_flags));
}
/// Supervisor memory management fence for all address spaces and virtual addresses
@ -129,7 +136,7 @@ pub unsafe fn sfence_vma_asid(asid: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn sfence_vma_all() {
asm!("sfence.vma", options(nostack))
asm!("sfence.vma", options(nostack, preserves_flags));
}
/// Invalidate supervisor translation cache for given virtual address and address space
@ -139,8 +146,13 @@ pub unsafe fn sfence_vma_all() {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn sinval_vma(vaddr: usize, asid: usize) {
// asm!("sinval.vma {}, {}", in(reg) vaddr, in(reg) asid, options(nostack))
asm!(".insn r 0x73, 0, 0x0B, x0, {}, {}", in(reg) vaddr, in(reg) asid, options(nostack))
// asm!("sinval.vma {}, {}", in(reg) vaddr, in(reg) asid, options(nostack, preserves_flags));
asm!(
".insn r 0x73, 0, 0x0B, x0, {}, {}",
in(reg) vaddr,
in(reg) asid,
options(nostack, preserves_flags)
);
}
/// Invalidate supervisor translation cache for given virtual address
@ -150,7 +162,11 @@ pub unsafe fn sinval_vma(vaddr: usize, asid: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn sinval_vma_vaddr(vaddr: usize) {
asm!(".insn r 0x73, 0, 0x0B, x0, {}, x0", in(reg) vaddr, options(nostack))
asm!(
".insn r 0x73, 0, 0x0B, x0, {}, x0",
in(reg) vaddr,
options(nostack, preserves_flags)
);
}
/// Invalidate supervisor translation cache for given address space
@ -160,7 +176,11 @@ pub unsafe fn sinval_vma_vaddr(vaddr: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn sinval_vma_asid(asid: usize) {
asm!(".insn r 0x73, 0, 0x0B, x0, x0, {}", in(reg) asid, options(nostack))
asm!(
".insn r 0x73, 0, 0x0B, x0, x0, {}",
in(reg) asid,
options(nostack, preserves_flags)
);
}
/// Invalidate supervisor translation cache for all address spaces and virtual addresses
@ -170,7 +190,10 @@ pub unsafe fn sinval_vma_asid(asid: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn sinval_vma_all() {
asm!(".insn r 0x73, 0, 0x0B, x0, x0, x0", options(nostack))
asm!(
".insn r 0x73, 0, 0x0B, x0, x0, x0",
options(nostack, preserves_flags)
);
}
/// Generates the `SFENCE.W.INVAL` instruction
@ -180,8 +203,11 @@ pub unsafe fn sinval_vma_all() {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn sfence_w_inval() {
// asm!("sfence.w.inval", options(nostack))
asm!(".insn i 0x73, 0, x0, x0, 0x180", options(nostack))
// asm!("sfence.w.inval", options(nostack, preserves_flags));
asm!(
".insn i 0x73, 0, x0, x0, 0x180",
options(nostack, preserves_flags)
);
}
/// Generates the `SFENCE.INVAL.IR` instruction
@ -191,8 +217,11 @@ pub unsafe fn sfence_w_inval() {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn sfence_inval_ir() {
// asm!("sfence.inval.ir", options(nostack))
asm!(".insn i 0x73, 0, x0, x0, 0x181", options(nostack))
// asm!("sfence.inval.ir", options(nostack, preserves_flags));
asm!(
".insn i 0x73, 0, x0, x0, 0x181",
options(nostack, preserves_flags)
);
}
/// Loads virtual machine memory by signed byte integer
@ -207,7 +236,12 @@ pub unsafe fn sfence_inval_ir() {
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hlv_b(src: *const i8) -> i8 {
let value: i8;
asm!(".insn i 0x73, 0x4, {}, {}, 0x600", out(reg) value, in(reg) src, options(readonly, nostack));
asm!(
".insn i 0x73, 0x4, {}, {}, 0x600",
lateout(reg) value,
in(reg) src,
options(readonly, nostack, preserves_flags)
);
value
}
@ -223,7 +257,12 @@ pub unsafe fn hlv_b(src: *const i8) -> i8 {
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hlv_bu(src: *const u8) -> u8 {
let value: u8;
asm!(".insn i 0x73, 0x4, {}, {}, 0x601", out(reg) value, in(reg) src, options(readonly, nostack));
asm!(
".insn i 0x73, 0x4, {}, {}, 0x601",
lateout(reg) value,
in(reg) src,
options(readonly, nostack, preserves_flags)
);
value
}
@ -239,7 +278,12 @@ pub unsafe fn hlv_bu(src: *const u8) -> u8 {
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hlv_h(src: *const i16) -> i16 {
let value: i16;
asm!(".insn i 0x73, 0x4, {}, {}, 0x640", out(reg) value, in(reg) src, options(readonly, nostack));
asm!(
".insn i 0x73, 0x4, {}, {}, 0x640",
lateout(reg) value,
in(reg) src,
options(readonly, nostack, preserves_flags)
);
value
}
@ -255,7 +299,12 @@ pub unsafe fn hlv_h(src: *const i16) -> i16 {
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hlv_hu(src: *const u16) -> u16 {
let value: u16;
asm!(".insn i 0x73, 0x4, {}, {}, 0x641", out(reg) value, in(reg) src, options(readonly, nostack));
asm!(
".insn i 0x73, 0x4, {}, {}, 0x641",
lateout(reg) value,
in(reg) src,
options(readonly, nostack, preserves_flags)
);
value
}
@ -271,7 +320,12 @@ pub unsafe fn hlv_hu(src: *const u16) -> u16 {
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hlvx_hu(src: *const u16) -> u16 {
let insn: u16;
asm!(".insn i 0x73, 0x4, {}, {}, 0x643", out(reg) insn, in(reg) src, options(readonly, nostack));
asm!(
".insn i 0x73, 0x4, {}, {}, 0x643",
lateout(reg) insn,
in(reg) src,
options(readonly, nostack, preserves_flags)
);
insn
}
@ -287,7 +341,12 @@ pub unsafe fn hlvx_hu(src: *const u16) -> u16 {
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hlv_w(src: *const i32) -> i32 {
let value: i32;
asm!(".insn i 0x73, 0x4, {}, {}, 0x680", out(reg) value, in(reg) src, options(readonly, nostack));
asm!(
".insn i 0x73, 0x4, {}, {}, 0x680",
lateout(reg) value,
in(reg) src,
options(readonly, nostack, preserves_flags)
);
value
}
@ -303,7 +362,12 @@ pub unsafe fn hlv_w(src: *const i32) -> i32 {
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hlvx_wu(src: *const u32) -> u32 {
let insn: u32;
asm!(".insn i 0x73, 0x4, {}, {}, 0x683", out(reg) insn, in(reg) src, options(readonly, nostack));
asm!(
".insn i 0x73, 0x4, {}, {}, 0x683",
lateout(reg) insn,
in(reg) src,
options(readonly, nostack, preserves_flags)
);
insn
}
@ -318,7 +382,12 @@ pub unsafe fn hlvx_wu(src: *const u32) -> u32 {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hsv_b(dst: *mut i8, src: i8) {
asm!(".insn r 0x73, 0x4, 0x31, x0, {}, {}", in(reg) dst, in(reg) src, options(nostack));
asm!(
".insn r 0x73, 0x4, 0x31, x0, {}, {}",
in(reg) dst,
in(reg) src,
options(nostack, preserves_flags)
);
}
/// Stores virtual machine memory by half integer
@ -332,7 +401,12 @@ pub unsafe fn hsv_b(dst: *mut i8, src: i8) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hsv_h(dst: *mut i16, src: i16) {
asm!(".insn r 0x73, 0x4, 0x33, x0, {}, {}", in(reg) dst, in(reg) src, options(nostack));
asm!(
".insn r 0x73, 0x4, 0x33, x0, {}, {}",
in(reg) dst,
in(reg) src,
options(nostack, preserves_flags)
);
}
/// Stores virtual machine memory by word integer
@ -346,7 +420,12 @@ pub unsafe fn hsv_h(dst: *mut i16, src: i16) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hsv_w(dst: *mut i32, src: i32) {
asm!(".insn r 0x73, 0x4, 0x35, x0, {}, {}", in(reg) dst, in(reg) src, options(nostack));
asm!(
".insn r 0x73, 0x4, 0x35, x0, {}, {}",
in(reg) dst,
in(reg) src,
options(nostack, preserves_flags)
);
}
/// Hypervisor memory management fence for given guest virtual address and guest address space
@ -360,8 +439,13 @@ pub unsafe fn hsv_w(dst: *mut i32, src: i32) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hfence_vvma(vaddr: usize, asid: usize) {
// asm!("hfence.vvma {}, {}", in(reg) vaddr, in(reg) asid)
asm!(".insn r 0x73, 0, 0x11, x0, {}, {}", in(reg) vaddr, in(reg) asid, options(nostack))
// asm!("hfence.vvma {}, {}", in(reg) vaddr, in(reg) asid, options(nostack, preserves_flags));
asm!(
".insn r 0x73, 0, 0x11, x0, {}, {}",
in(reg) vaddr,
in(reg) asid,
options(nostack, preserves_flags)
);
}
/// Hypervisor memory management fence for given guest virtual address
@ -375,7 +459,11 @@ pub unsafe fn hfence_vvma(vaddr: usize, asid: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hfence_vvma_vaddr(vaddr: usize) {
asm!(".insn r 0x73, 0, 0x11, x0, {}, x0", in(reg) vaddr, options(nostack))
asm!(
".insn r 0x73, 0, 0x11, x0, {}, x0",
in(reg) vaddr,
options(nostack, preserves_flags)
);
}
/// Hypervisor memory management fence for given guest address space
@ -389,7 +477,11 @@ pub unsafe fn hfence_vvma_vaddr(vaddr: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hfence_vvma_asid(asid: usize) {
asm!(".insn r 0x73, 0, 0x11, x0, x0, {}", in(reg) asid, options(nostack))
asm!(
".insn r 0x73, 0, 0x11, x0, x0, {}",
in(reg) asid,
options(nostack, preserves_flags)
);
}
/// Hypervisor memory management fence for all guest address spaces and guest virtual addresses
@ -403,7 +495,10 @@ pub unsafe fn hfence_vvma_asid(asid: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hfence_vvma_all() {
asm!(".insn r 0x73, 0, 0x11, x0, x0, x0", options(nostack))
asm!(
".insn r 0x73, 0, 0x11, x0, x0, x0",
options(nostack, preserves_flags)
);
}
/// Hypervisor memory management fence for guest physical address and virtual machine
@ -416,8 +511,13 @@ pub unsafe fn hfence_vvma_all() {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hfence_gvma(gaddr: usize, vmid: usize) {
// asm!("hfence.gvma {}, {}", in(reg) gaddr, in(reg) vmid, options(nostack))
asm!(".insn r 0x73, 0, 0x31, x0, {}, {}", in(reg) gaddr, in(reg) vmid, options(nostack))
// asm!("hfence.gvma {}, {}", in(reg) gaddr, in(reg) vmid, options(nostack, preserves_flags));
asm!(
".insn r 0x73, 0, 0x31, x0, {}, {}",
in(reg) gaddr,
in(reg) vmid,
options(nostack, preserves_flags)
);
}
/// Hypervisor memory management fence for guest physical address
@ -429,7 +529,11 @@ pub unsafe fn hfence_gvma(gaddr: usize, vmid: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hfence_gvma_gaddr(gaddr: usize) {
asm!(".insn r 0x73, 0, 0x31, x0, {}, x0", in(reg) gaddr, options(nostack))
asm!(
".insn r 0x73, 0, 0x31, x0, {}, x0",
in(reg) gaddr,
options(nostack, preserves_flags)
);
}
/// Hypervisor memory management fence for given virtual machine
@ -441,7 +545,11 @@ pub unsafe fn hfence_gvma_gaddr(gaddr: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hfence_gvma_vmid(vmid: usize) {
asm!(".insn r 0x73, 0, 0x31, x0, x0, {}", in(reg) vmid, options(nostack))
asm!(
".insn r 0x73, 0, 0x31, x0, x0, {}",
in(reg) vmid,
options(nostack, preserves_flags)
);
}
/// Hypervisor memory management fence for all virtual machines and guest physical addresses
@ -453,7 +561,10 @@ pub unsafe fn hfence_gvma_vmid(vmid: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hfence_gvma_all() {
asm!(".insn r 0x73, 0, 0x31, x0, x0, x0", options(nostack))
asm!(
".insn r 0x73, 0, 0x31, x0, x0, x0",
options(nostack, preserves_flags)
);
}
/// Invalidate hypervisor translation cache for given guest virtual address and guest address space
@ -465,8 +576,13 @@ pub unsafe fn hfence_gvma_all() {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hinval_vvma(vaddr: usize, asid: usize) {
// asm!("hinval.vvma {}, {}", in(reg) vaddr, in(reg) asid, options(nostack))
asm!(".insn r 0x73, 0, 0x13, x0, {}, {}", in(reg) vaddr, in(reg) asid, options(nostack))
// asm!("hinval.vvma {}, {}", in(reg) vaddr, in(reg) asid, options(nostack, preserves_flags));
asm!(
".insn r 0x73, 0, 0x13, x0, {}, {}",
in(reg) vaddr,
in(reg) asid,
options(nostack, preserves_flags)
);
}
/// Invalidate hypervisor translation cache for given guest virtual address
@ -478,7 +594,11 @@ pub unsafe fn hinval_vvma(vaddr: usize, asid: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hinval_vvma_vaddr(vaddr: usize) {
asm!(".insn r 0x73, 0, 0x13, x0, {}, x0", in(reg) vaddr, options(nostack))
asm!(
".insn r 0x73, 0, 0x13, x0, {}, x0",
in(reg) vaddr,
options(nostack, preserves_flags)
);
}
/// Invalidate hypervisor translation cache for given guest address space
@ -490,7 +610,11 @@ pub unsafe fn hinval_vvma_vaddr(vaddr: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hinval_vvma_asid(asid: usize) {
asm!(".insn r 0x73, 0, 0x13, x0, x0, {}", in(reg) asid, options(nostack))
asm!(
".insn r 0x73, 0, 0x13, x0, x0, {}",
in(reg) asid,
options(nostack, preserves_flags)
);
}
/// Invalidate hypervisor translation cache for all guest address spaces and guest virtual addresses
@ -502,7 +626,10 @@ pub unsafe fn hinval_vvma_asid(asid: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hinval_vvma_all() {
asm!(".insn r 0x73, 0, 0x13, x0, x0, x0", options(nostack))
asm!(
".insn r 0x73, 0, 0x13, x0, x0, x0",
options(nostack, preserves_flags)
);
}
/// Invalidate hypervisor translation cache for guest physical address and virtual machine
@ -515,8 +642,13 @@ pub unsafe fn hinval_vvma_all() {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hinval_gvma(gaddr: usize, vmid: usize) {
// asm!("hinval.gvma {}, {}", in(reg) gaddr, in(reg) vmid, options(nostack))
asm!(".insn r 0x73, 0, 0x33, x0, {}, {}", in(reg) gaddr, in(reg) vmid, options(nostack))
// asm!("hinval.gvma {}, {}", in(reg) gaddr, in(reg) vmid, options(nostack, preserves_flags));
asm!(
".insn r 0x73, 0, 0x33, x0, {}, {}",
in(reg) gaddr,
in(reg) vmid,
options(nostack, preserves_flags)
);
}
/// Invalidate hypervisor translation cache for guest physical address
@ -528,7 +660,11 @@ pub unsafe fn hinval_gvma(gaddr: usize, vmid: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hinval_gvma_gaddr(gaddr: usize) {
asm!(".insn r 0x73, 0, 0x33, x0, {}, x0", in(reg) gaddr, options(nostack))
asm!(
".insn r 0x73, 0, 0x33, x0, {}, x0",
in(reg) gaddr,
options(nostack, preserves_flags)
);
}
/// Invalidate hypervisor translation cache for given virtual machine
@ -540,7 +676,11 @@ pub unsafe fn hinval_gvma_gaddr(gaddr: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hinval_gvma_vmid(vmid: usize) {
asm!(".insn r 0x73, 0, 0x33, x0, x0, {}", in(reg) vmid, options(nostack))
asm!(
".insn r 0x73, 0, 0x33, x0, x0, {}",
in(reg) vmid,
options(nostack, preserves_flags)
);
}
/// Invalidate hypervisor translation cache for all virtual machines and guest physical addresses
@ -552,7 +692,10 @@ pub unsafe fn hinval_gvma_vmid(vmid: usize) {
#[inline]
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub unsafe fn hinval_gvma_all() {
asm!(".insn r 0x73, 0, 0x33, x0, x0, x0", options(nostack))
asm!(
".insn r 0x73, 0, 0x33, x0, x0, x0",
options(nostack, preserves_flags)
);
}
/// Reads the floating-point rounding mode register `frm`
@ -574,6 +717,12 @@ pub unsafe fn hinval_gvma_all() {
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
pub fn frrm() -> u32 {
let value: u32;
unsafe { asm!("frrm {}", out(reg) value, options(nomem, nostack)) };
unsafe {
asm!(
"frrm {}",
out(reg) value,
options(nomem, nostack, preserves_flags)
);
}
value
}

4
library/stdarch/crates/core_arch/src/riscv_shared/zb.rs
View file

@ -68,7 +68,7 @@ pub fn orc_b(rs: usize) -> usize {
///
/// Section: 2.11
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
#[target_feature(enable = "zbc")]
#[target_feature(enable = "zbkc")]
#[cfg_attr(test, assert_instr(clmul))]
#[inline]
pub fn clmul(rs1: usize, rs2: usize) -> usize {
@ -93,7 +93,7 @@ pub fn clmul(rs1: usize, rs2: usize) -> usize {
///
/// Section: 2.12
#[unstable(feature = "riscv_ext_intrinsics", issue = "114544")]
#[target_feature(enable = "zbc")]
#[target_feature(enable = "zbkc")]
#[cfg_attr(test, assert_instr(clmulh))]
#[inline]
pub fn clmulh(rs1: usize, rs2: usize) -> usize {

225
library/stdarch/crates/core_arch/src/s390x/vector.rs
View file

@ -94,8 +94,6 @@ unsafe extern "unadjusted" {
#[link_name = "llvm.s390.vsrlb"] fn vsrlb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
#[link_name = "llvm.s390.vslb"] fn vslb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
#[link_name = "llvm.s390.vsldb"] fn vsldb(a: i8x16, b: i8x16, c: u32) -> i8x16;
#[link_name = "llvm.s390.vsld"] fn vsld(a: i8x16, b: i8x16, c: u32) -> i8x16;
#[link_name = "llvm.s390.vsrd"] fn vsrd(a: i8x16, b: i8x16, c: u32) -> i8x16;
#[link_name = "llvm.s390.verimb"] fn verimb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char, d: i32) -> vector_signed_char;
@ -114,6 +112,9 @@ unsafe extern "unadjusted" {
#[link_name = "llvm.s390.vsumqf"] fn vsumqf(a: vector_unsigned_int, b: vector_unsigned_int) -> u128;
#[link_name = "llvm.s390.vsumqg"] fn vsumqg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128;
#[link_name = "llvm.s390.vaccq"] fn vaccq(a: u128, b: u128) -> u128;
#[link_name = "llvm.s390.vacccq"] fn vacccq(a: u128, b: u128, c: u128) -> u128;
#[link_name = "llvm.s390.vscbiq"] fn vscbiq(a: u128, b: u128) -> u128;
#[link_name = "llvm.s390.vsbiq"] fn vsbiq(a: u128, b: u128, c: u128) -> u128;
#[link_name = "llvm.s390.vsbcbiq"] fn vsbcbiq(a: u128, b: u128, c: u128) -> u128;
@ -166,13 +167,6 @@ unsafe extern "unadjusted" {
#[link_name = "llvm.s390.vpklsfs"] fn vpklsfs(a: vector_unsigned_int, b: vector_unsigned_int) -> PackedTuple<vector_unsigned_short, i32>;
#[link_name = "llvm.s390.vpklsgs"] fn vpklsgs(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> PackedTuple<vector_unsigned_int, i32>;
#[link_name = "llvm.s390.vuplb"] fn vuplb (a: vector_signed_char) -> vector_signed_short;
#[link_name = "llvm.s390.vuplhw"] fn vuplhw (a: vector_signed_short) -> vector_signed_int;
#[link_name = "llvm.s390.vuplf"] fn vuplf (a: vector_signed_int) -> vector_signed_long_long;
#[link_name = "llvm.s390.vupllb"] fn vupllb (a: vector_unsigned_char) -> vector_unsigned_short;
#[link_name = "llvm.s390.vupllh"] fn vupllh (a: vector_unsigned_short) -> vector_unsigned_int;
#[link_name = "llvm.s390.vupllf"] fn vupllf (a: vector_unsigned_int) -> vector_unsigned_long_long;
#[link_name = "llvm.s390.vavgb"] fn vavgb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
#[link_name = "llvm.s390.vavgh"] fn vavgh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
#[link_name = "llvm.s390.vavgf"] fn vavgf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
@ -185,22 +179,6 @@ unsafe extern "unadjusted" {
#[link_name = "llvm.s390.vcksm"] fn vcksm(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
#[link_name = "llvm.s390.vmeb"] fn vmeb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short;
#[link_name = "llvm.s390.vmeh"] fn vmeh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int;
#[link_name = "llvm.s390.vmef"] fn vmef(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long;
#[link_name = "llvm.s390.vmleb"] fn vmleb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
#[link_name = "llvm.s390.vmleh"] fn vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
#[link_name = "llvm.s390.vmlef"] fn vmlef(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
#[link_name = "llvm.s390.vmob"] fn vmob(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short;
#[link_name = "llvm.s390.vmoh"] fn vmoh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int;
#[link_name = "llvm.s390.vmof"] fn vmof(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long;
#[link_name = "llvm.s390.vmlob"] fn vmlob(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
#[link_name = "llvm.s390.vmloh"] fn vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
#[link_name = "llvm.s390.vmlof"] fn vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
#[link_name = "llvm.s390.vmhb"] fn vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
#[link_name = "llvm.s390.vmhh"] fn vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
#[link_name = "llvm.s390.vmhf"] fn vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
@ -376,7 +354,20 @@ impl<const N: usize> ShuffleMask<N> {
ShuffleMask(mask)
}
const fn pack() -> Self {
const fn even() -> Self {
let mut mask = [0; N];
let mut i = 0;
let mut index = 0;
while index < N {
mask[index] = i as u32;
i += 2;
index += 1;
}
ShuffleMask(mask)
}
const fn odd() -> Self {
let mut mask = [0; N];
let mut i = 1;
let mut index = 0;
@ -389,6 +380,10 @@ impl<const N: usize> ShuffleMask<N> {
ShuffleMask(mask)
}
const fn pack() -> Self {
Self::odd()
}
const fn unpack_low() -> Self {
let mut mask = [0; N];
let mut i = 0;
@ -1198,10 +1193,8 @@ mod sealed {
test_impl! { vec_roundc_f32 (a: vector_float) -> vector_float [nearbyint_v4f32, "vector-enhancements-1" vfisb] }
test_impl! { vec_roundc_f64 (a: vector_double) -> vector_double [nearbyint_v2f64, vfidb] }
// FIXME(llvm) llvm trunk already lowers roundeven to vfidb, but rust does not use it yet
// use https://godbolt.org/z/cWq95fexe to check, and enable the instruction test when it works
test_impl! { vec_round_f32 (a: vector_float) -> vector_float [roundeven_v4f32, _] }
test_impl! { vec_round_f64 (a: vector_double) -> vector_double [roundeven_v2f64, _] }
test_impl! { vec_round_f32 (a: vector_float) -> vector_float [roundeven_v4f32, "vector-enhancements-1" vfisb] }
test_impl! { vec_round_f64 (a: vector_double) -> vector_double [roundeven_v2f64, vfidb] }
#[unstable(feature = "stdarch_s390x", issue = "135681")]
pub trait VectorRoundc {
@ -2359,6 +2352,9 @@ mod sealed {
unsafe fn vec_packs(self, b: Other) -> Self::Result;
}
// FIXME(llvm): https://github.com/llvm/llvm-project/issues/153655
// Other targets can use a min/max for the saturation + a truncation.
impl_vec_trait! { [VectorPacks vec_packs] vpksh (vector_signed_short, vector_signed_short) -> vector_signed_char }
impl_vec_trait! { [VectorPacks vec_packs] vpklsh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
impl_vec_trait! { [VectorPacks vec_packs] vpksf (vector_signed_int, vector_signed_int) -> vector_signed_short }
@ -2580,8 +2576,14 @@ mod sealed {
unsafe fn vec_unpackl(self) -> Self::Result;
}
// FIXME(llvm): a shuffle + simd_as does not currently optimize into a single instruction like
// unpachk above. Tracked in https://github.com/llvm/llvm-project/issues/129576.
// NOTE: `vuplh` is used for "unpack logical high", hence `vuplhw`.
impl_vec_unpack!(unpack_low vuplb vector_signed_char i8x8 vector_signed_short 8);
impl_vec_unpack!(unpack_low vuplhw vector_signed_short i16x4 vector_signed_int 4);
impl_vec_unpack!(unpack_low vuplf vector_signed_int i32x2 vector_signed_long_long 2);
impl_vec_unpack!(unpack_low vupllb vector_unsigned_char u8x8 vector_unsigned_short 8);
impl_vec_unpack!(unpack_low vupllh vector_unsigned_short u16x4 vector_unsigned_int 4);
impl_vec_unpack!(unpack_low vupllf vector_unsigned_int u32x2 vector_unsigned_long_long 2);
impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplb (vector_signed_char) -> vector_signed_short}
impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplhw (vector_signed_short) -> vector_signed_int}
@ -2642,61 +2644,65 @@ mod sealed {
unsafe fn vec_mule(self, b: Self) -> Result;
}
// FIXME(llvm) sadly this does not yet work https://github.com/llvm/llvm-project/issues/129705
// #[target_feature(enable = "vector")]
// #[cfg_attr(test, assert_instr(vmleh))]
// unsafe fn vec_vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int {
// let even_a: vector_unsigned_int = simd_as(simd_shuffle::<_, _, u16x4>(
// a,
// a,
// const { ShuffleMask([0, 2, 4, 6]) },
// ));
//
// let even_b: vector_unsigned_int = simd_as(simd_shuffle::<_, _, u16x4>(
// b,
// b,
// const { ShuffleMask([0, 2, 4, 6]) },
// ));
//
// simd_mul(even_a, even_b)
// }
macro_rules! impl_vec_mul_even_odd {
($mask:ident $instr:ident $src:ident $shuffled:ident $dst:ident $width:literal) => {
#[inline]
#[target_feature(enable = "vector")]
#[cfg_attr(test, assert_instr($instr))]
unsafe fn $instr(a: $src, b: $src) -> $dst {
let elems_a: $dst = simd_as(simd_shuffle::<_, _, $shuffled>(
a,
a, // this argument is ignored entirely.
const { ShuffleMask::<$width>::$mask() },
));
test_impl! { vec_vmeb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short [ vmeb, vmeb ] }
test_impl! { vec_vmeh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int[ vmeh, vmeh ] }
test_impl! { vec_vmef(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long [ vmef, vmef ] }
let elems_b: $dst = simd_as(simd_shuffle::<_, _, $shuffled>(
b,
b, // this argument is ignored entirely.
const { ShuffleMask::<$width>::$mask() },
));
test_impl! { vec_vmleb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vmleb, vmleb ] }
test_impl! { vec_vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vmleh, vmleh ] }
test_impl! { vec_vmlef(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vmlef, vmlef ] }
simd_mul(elems_a, elems_b)
}
};
}
impl_mul!([VectorMule vec_mule] vec_vmeb (vector_signed_char, vector_signed_char) -> vector_signed_short );
impl_mul!([VectorMule vec_mule] vec_vmeh (vector_signed_short, vector_signed_short) -> vector_signed_int);
impl_mul!([VectorMule vec_mule] vec_vmef (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
impl_vec_mul_even_odd! { even vmeb vector_signed_char i8x8 vector_signed_short 8 }
impl_vec_mul_even_odd! { even vmeh vector_signed_short i16x4 vector_signed_int 4 }
impl_vec_mul_even_odd! { even vmef vector_signed_int i32x2 vector_signed_long_long 2 }
impl_mul!([VectorMule vec_mule] vec_vmleb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
impl_mul!([VectorMule vec_mule] vec_vmleh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
impl_mul!([VectorMule vec_mule] vec_vmlef (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
impl_vec_mul_even_odd! { even vmleb vector_unsigned_char u8x8 vector_unsigned_short 8 }
impl_vec_mul_even_odd! { even vmleh vector_unsigned_short u16x4 vector_unsigned_int 4 }
impl_vec_mul_even_odd! { even vmlef vector_unsigned_int u32x2 vector_unsigned_long_long 2 }
impl_mul!([VectorMule vec_mule] vmeb (vector_signed_char, vector_signed_char) -> vector_signed_short );
impl_mul!([VectorMule vec_mule] vmeh (vector_signed_short, vector_signed_short) -> vector_signed_int);
impl_mul!([VectorMule vec_mule] vmef (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
impl_mul!([VectorMule vec_mule] vmleb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
impl_mul!([VectorMule vec_mule] vmleh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
impl_mul!([VectorMule vec_mule] vmlef (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
#[unstable(feature = "stdarch_s390x", issue = "135681")]
pub trait VectorMulo<Result> {
unsafe fn vec_mulo(self, b: Self) -> Result;
}
test_impl! { vec_vmob(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short [ vmob, vmob ] }
test_impl! { vec_vmoh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int[ vmoh, vmoh ] }
test_impl! { vec_vmof(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long [ vmof, vmof ] }
impl_vec_mul_even_odd! { odd vmob vector_signed_char i8x8 vector_signed_short 8 }
impl_vec_mul_even_odd! { odd vmoh vector_signed_short i16x4 vector_signed_int 4 }
impl_vec_mul_even_odd! { odd vmof vector_signed_int i32x2 vector_signed_long_long 2 }
test_impl! { vec_vmlob(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vmlob, vmlob ] }
test_impl! { vec_vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vmloh, vmloh ] }
test_impl! { vec_vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vmlof, vmlof ] }
impl_vec_mul_even_odd! { odd vmlob vector_unsigned_char u8x8 vector_unsigned_short 8 }
impl_vec_mul_even_odd! { odd vmloh vector_unsigned_short u16x4 vector_unsigned_int 4 }
impl_vec_mul_even_odd! { odd vmlof vector_unsigned_int u32x2 vector_unsigned_long_long 2 }
impl_mul!([VectorMulo vec_mulo] vec_vmob (vector_signed_char, vector_signed_char) -> vector_signed_short );
impl_mul!([VectorMulo vec_mulo] vec_vmoh (vector_signed_short, vector_signed_short) -> vector_signed_int);
impl_mul!([VectorMulo vec_mulo] vec_vmof (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
impl_mul!([VectorMulo vec_mulo] vmob (vector_signed_char, vector_signed_char) -> vector_signed_short );
impl_mul!([VectorMulo vec_mulo] vmoh (vector_signed_short, vector_signed_short) -> vector_signed_int);
impl_mul!([VectorMulo vec_mulo] vmof (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
impl_mul!([VectorMulo vec_mulo] vec_vmlob (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
impl_mul!([VectorMulo vec_mulo] vec_vmloh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
impl_mul!([VectorMulo vec_mulo] vec_vmlof (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
impl_mul!([VectorMulo vec_mulo] vmlob (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
impl_mul!([VectorMulo vec_mulo] vmloh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
impl_mul!([VectorMulo vec_mulo] vmlof (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
#[unstable(feature = "stdarch_s390x", issue = "135681")]
pub trait VectorMulh<Result> {
@ -3319,8 +3325,7 @@ mod sealed {
#[inline]
#[target_feature(enable = "vector")]
// FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
// #[cfg_attr(test, assert_instr(vsegb))]
#[cfg_attr(test, assert_instr(vsegb))]
pub unsafe fn vsegb(a: vector_signed_char) -> vector_signed_long_long {
simd_as(simd_shuffle::<_, _, i8x2>(
a,
@ -3331,8 +3336,7 @@ mod sealed {
#[inline]
#[target_feature(enable = "vector")]
// FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
// #[cfg_attr(test, assert_instr(vsegh))]
#[cfg_attr(test, assert_instr(vsegh))]
pub unsafe fn vsegh(a: vector_signed_short) -> vector_signed_long_long {
simd_as(simd_shuffle::<_, _, i16x2>(
a,
@ -3343,8 +3347,7 @@ mod sealed {
#[inline]
#[target_feature(enable = "vector")]
// FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
// #[cfg_attr(test, assert_instr(vsegf))]
#[cfg_attr(test, assert_instr(vsegf))]
pub unsafe fn vsegf(a: vector_signed_int) -> vector_signed_long_long {
simd_as(simd_shuffle::<_, _, i32x2>(
a,
@ -3482,10 +3485,33 @@ mod sealed {
unsafe fn vec_sldb<const C: u32>(self, b: Self) -> Self;
}
// FIXME(llvm) https://github.com/llvm/llvm-project/issues/129955
// ideally we could implement this in terms of llvm.fshl.i128
// #[link_name = "llvm.fshl.i128"] fn fshl_i128(a: u128, b: u128, c: u128) -> u128;
// transmute(fshl_i128(transmute(a), transmute(b), const { C * 8 } ))
#[inline]
#[target_feature(enable = "vector")]
#[cfg_attr(test, assert_instr(vsldb))]
unsafe fn test_vec_sld(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int {
a.vec_sld::<13>(b)
}
#[inline]
#[target_feature(enable = "vector")]
#[cfg_attr(test, assert_instr(vsldb))]
unsafe fn test_vec_sldw(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int {
a.vec_sldw::<3>(b)
}
#[inline]
#[target_feature(enable = "vector-enhancements-2")]
#[cfg_attr(test, assert_instr(vsld))]
unsafe fn test_vec_sldb(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int {
a.vec_sldb::<7>(b)
}
#[inline]
#[target_feature(enable = "vector-enhancements-2")]
#[cfg_attr(test, assert_instr(vsrd))]
unsafe fn test_vec_srdb(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int {
a.vec_srdb::<7>(b)
}
macro_rules! impl_vec_sld {
($($ty:ident)*) => {
@ -3496,21 +3522,21 @@ mod sealed {
#[target_feature(enable = "vector")]
unsafe fn vec_sld<const C: u32>(self, b: Self) -> Self {
static_assert_uimm_bits!(C, 4);
transmute(vsldb(transmute(self), transmute(b), C))
transmute(u128::funnel_shl(transmute(self), transmute(b), C * 8))
}
#[inline]
#[target_feature(enable = "vector")]
unsafe fn vec_sldw<const C: u32>(self, b: Self) -> Self {
static_assert_uimm_bits!(C, 2);
transmute(vsldb(transmute(self), transmute(b), const { 4 * C }))
transmute(u128::funnel_shl(transmute(self), transmute(b), C * 4 * 8))
}
#[inline]
#[target_feature(enable = "vector-enhancements-2")]
unsafe fn vec_sldb<const C: u32>(self, b: Self) -> Self {
static_assert_uimm_bits!(C, 3);
transmute(vsld(transmute(self), transmute(b), C))
transmute(u128::funnel_shl(transmute(self), transmute(b), C))
}
}
@ -3521,6 +3547,11 @@ mod sealed {
unsafe fn vec_srdb<const C: u32>(self, b: Self) -> Self {
static_assert_uimm_bits!(C, 3);
transmute(vsrd(transmute(self), transmute(b), C))
// FIXME(llvm): https://github.com/llvm/llvm-project/issues/129955#issuecomment-3207488190
// LLVM currently rewrites `fshr` to `fshl`, and the logic in the s390x
// backend cannot deal with that yet.
// #[link_name = "llvm.fshr.i128"] fn fshr_i128(a: u128, b: u128, c: u128) -> u128;
// transmute(fshr_i128(transmute(self), transmute(b), const { C as u128 }))
}
}
)*
@ -4676,11 +4707,9 @@ pub unsafe fn vec_subc_u128(
a: vector_unsigned_char,
b: vector_unsigned_char,
) -> vector_unsigned_char {
// FIXME(llvm) sadly this does not work https://github.com/llvm/llvm-project/issues/129608
// let a: u128 = transmute(a);
// let b: u128 = transmute(b);
// transmute(!a.overflowing_sub(b).1 as u128)
transmute(vscbiq(transmute(a), transmute(b)))
let a: u128 = transmute(a);
let b: u128 = transmute(b);
transmute(!a.overflowing_sub(b).1 as u128)
}
/// Vector Add Compute Carryout unsigned 128-bits
@ -4694,7 +4723,9 @@ pub unsafe fn vec_addc_u128(
) -> vector_unsigned_char {
let a: u128 = transmute(a);
let b: u128 = transmute(b);
transmute(a.overflowing_add(b).1 as u128)
// FIXME(llvm) https://github.com/llvm/llvm-project/pull/153557
// transmute(a.overflowing_add(b).1 as u128)
transmute(vaccq(a, b))
}
/// Vector Add With Carry unsigned 128-bits
@ -4710,7 +4741,7 @@ pub unsafe fn vec_adde_u128(
let a: u128 = transmute(a);
let b: u128 = transmute(b);
let c: u128 = transmute(c);
// FIXME(llvm) sadly this does not work
// FIXME(llvm) https://github.com/llvm/llvm-project/pull/153557
// let (d, _carry) = a.carrying_add(b, c & 1 != 0);
// transmute(d)
transmute(vacq(a, b, c))
@ -4729,8 +4760,10 @@ pub unsafe fn vec_addec_u128(
let a: u128 = transmute(a);
let b: u128 = transmute(b);
let c: u128 = transmute(c);
let (_d, carry) = a.carrying_add(b, c & 1 != 0);
transmute(carry as u128)
// FIXME(llvm) https://github.com/llvm/llvm-project/pull/153557
// let (_d, carry) = a.carrying_add(b, c & 1 != 0);
// transmute(carry as u128)
transmute(vacccq(a, b, c))
}
/// Vector Subtract with Carryout

38
library/stdarch/crates/core_arch/src/wasm32/simd128.rs
View file

@ -141,16 +141,6 @@ unsafe extern "unadjusted" {
fn llvm_f64x2_max(x: simd::f64x2, y: simd::f64x2) -> simd::f64x2;
}
#[repr(C, packed)]
#[derive(Copy)]
struct Unaligned<T>(T);
impl<T: Copy> Clone for Unaligned<T> {
fn clone(&self) -> Unaligned<T> {
*self
}
}
/// Loads a `v128` vector from the given heap address.
///
/// This intrinsic will emit a load with an alignment of 1. While this is
@ -179,7 +169,7 @@ impl<T: Copy> Clone for Unaligned<T> {
#[doc(alias("v128.load"))]
#[stable(feature = "wasm_simd", since = "1.54.0")]
pub unsafe fn v128_load(m: *const v128) -> v128 {
(*(m as *const Unaligned<v128>)).0
m.read_unaligned()
}
/// Load eight 8-bit integers and sign extend each one to a 16-bit lane
@ -196,8 +186,8 @@ pub unsafe fn v128_load(m: *const v128) -> v128 {
#[doc(alias("v128.load8x8_s"))]
#[stable(feature = "wasm_simd", since = "1.54.0")]
pub unsafe fn i16x8_load_extend_i8x8(m: *const i8) -> v128 {
let m = *(m as *const Unaligned<simd::i8x8>);
simd_cast::<_, simd::i16x8>(m.0).v128()
let m = m.cast::<simd::i8x8>().read_unaligned();
simd_cast::<_, simd::i16x8>(m).v128()
}
/// Load eight 8-bit integers and zero extend each one to a 16-bit lane
@ -214,8 +204,8 @@ pub unsafe fn i16x8_load_extend_i8x8(m: *const i8) -> v128 {
#[doc(alias("v128.load8x8_u"))]
#[stable(feature = "wasm_simd", since = "1.54.0")]
pub unsafe fn i16x8_load_extend_u8x8(m: *const u8) -> v128 {
let m = *(m as *const Unaligned<simd::u8x8>);
simd_cast::<_, simd::u16x8>(m.0).v128()
let m = m.cast::<simd::u8x8>().read_unaligned();
simd_cast::<_, simd::u16x8>(m).v128()
}
#[stable(feature = "wasm_simd", since = "1.54.0")]
@ -235,8 +225,8 @@ pub use i16x8_load_extend_u8x8 as u16x8_load_extend_u8x8;
#[doc(alias("v128.load16x4_s"))]
#[stable(feature = "wasm_simd", since = "1.54.0")]
pub unsafe fn i32x4_load_extend_i16x4(m: *const i16) -> v128 {
let m = *(m as *const Unaligned<simd::i16x4>);
simd_cast::<_, simd::i32x4>(m.0).v128()
let m = m.cast::<simd::i16x4>().read_unaligned();
simd_cast::<_, simd::i32x4>(m).v128()
}
/// Load four 16-bit integers and zero extend each one to a 32-bit lane
@ -253,8 +243,8 @@ pub unsafe fn i32x4_load_extend_i16x4(m: *const i16) -> v128 {
#[doc(alias("v128.load16x4_u"))]
#[stable(feature = "wasm_simd", since = "1.54.0")]
pub unsafe fn i32x4_load_extend_u16x4(m: *const u16) -> v128 {
let m = *(m as *const Unaligned<simd::u16x4>);
simd_cast::<_, simd::u32x4>(m.0).v128()
let m = m.cast::<simd::u16x4>().read_unaligned();
simd_cast::<_, simd::u32x4>(m).v128()
}
#[stable(feature = "wasm_simd", since = "1.54.0")]
@ -274,8 +264,8 @@ pub use i32x4_load_extend_u16x4 as u32x4_load_extend_u16x4;
#[doc(alias("v128.load32x2_s"))]
#[stable(feature = "wasm_simd", since = "1.54.0")]
pub unsafe fn i64x2_load_extend_i32x2(m: *const i32) -> v128 {
let m = *(m as *const Unaligned<simd::i32x2>);
simd_cast::<_, simd::i64x2>(m.0).v128()
let m = m.cast::<simd::i32x2>().read_unaligned();
simd_cast::<_, simd::i64x2>(m).v128()
}
/// Load two 32-bit integers and zero extend each one to a 64-bit lane
@ -292,8 +282,8 @@ pub unsafe fn i64x2_load_extend_i32x2(m: *const i32) -> v128 {
#[doc(alias("v128.load32x2_u"))]
#[stable(feature = "wasm_simd", since = "1.54.0")]
pub unsafe fn i64x2_load_extend_u32x2(m: *const u32) -> v128 {
let m = *(m as *const Unaligned<simd::u32x2>);
simd_cast::<_, simd::u64x2>(m.0).v128()
let m = m.cast::<simd::u32x2>().read_unaligned();
simd_cast::<_, simd::u64x2>(m).v128()
}
#[stable(feature = "wasm_simd", since = "1.54.0")]
@ -453,7 +443,7 @@ pub unsafe fn v128_load64_zero(m: *const u64) -> v128 {
#[doc(alias("v128.store"))]
#[stable(feature = "wasm_simd", since = "1.54.0")]
pub unsafe fn v128_store(m: *mut v128, a: v128) {
*(m as *mut Unaligned<v128>) = Unaligned(a);
m.write_unaligned(a)
}
/// Loads an 8-bit value from `m` and sets lane `L` of `v` to that value.

4
library/stdarch/crates/core_arch/src/x86/avx512f.rs
View file

@ -5823,7 +5823,7 @@ pub fn _mm512_maskz_roundscale_pd<const IMM8: i32>(k: __mmask8, a: __m512d) -> _
#[inline]
#[target_feature(enable = "avx512f,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))]
#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 16))]
#[rustc_legacy_const_generics(1)]
pub fn _mm256_roundscale_pd<const IMM8: i32>(a: __m256d) -> __m256d {
unsafe {
@ -5897,7 +5897,7 @@ pub fn _mm256_maskz_roundscale_pd<const IMM8: i32>(k: __mmask8, a: __m256d) -> _
#[inline]
#[target_feature(enable = "avx512f,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 0))]
#[cfg_attr(test, assert_instr(vrndscalepd, IMM8 = 16))]
#[rustc_legacy_const_generics(1)]
pub fn _mm_roundscale_pd<const IMM8: i32>(a: __m128d) -> __m128d {
unsafe {

376
library/stdarch/crates/core_arch/src/x86/avx512fp16.rs
View file

File diff suppressed because it is too large Load diff

33
library/stdarch/crates/intrinsic-test/src/arm/config.rs
View file

@ -1,14 +1,17 @@
pub fn build_notices(line_prefix: &str) -> String {
format!(
"\
{line_prefix}This is a transient test file, not intended for distribution. Some aspects of the
{line_prefix}test are derived from a JSON specification, published under the same license as the
{line_prefix}`intrinsic-test` crate.\n
"
)
}
pub const NOTICE: &str = "\
// This is a transient test file, not intended for distribution. Some aspects of the
// test are derived from a JSON specification, published under the same license as the
// `intrinsic-test` crate.\n";
pub const POLY128_OSTREAM_DEF: &str = r#"std::ostream& operator<<(std::ostream& os, poly128_t value) {
pub const POLY128_OSTREAM_DECL: &str = r#"
#ifdef __aarch64__
std::ostream& operator<<(std::ostream& os, poly128_t value);
#endif
"#;
pub const POLY128_OSTREAM_DEF: &str = r#"
#ifdef __aarch64__
std::ostream& operator<<(std::ostream& os, poly128_t value) {
std::stringstream temp;
do {
int n = value % 10;
@ -19,7 +22,9 @@ pub const POLY128_OSTREAM_DEF: &str = r#"std::ostream& operator<<(std::ostream&
std::string res(tempstr.rbegin(), tempstr.rend());
os << res;
return os;
}"#;
}
#endif
"#;
// Format f16 values (and vectors containing them) in a way that is consistent with C.
pub const F16_FORMATTING_DEF: &str = r#"
@ -118,4 +123,10 @@ pub const AARCH_CONFIGURATIONS: &str = r#"
#![cfg_attr(any(target_arch = "aarch64", target_arch = "arm64ec"), feature(stdarch_neon_ftts))]
#![feature(fmt_helpers_for_derive)]
#![feature(stdarch_neon_f16)]
#[cfg(any(target_arch = "aarch64", target_arch = "arm64ec"))]
use core::arch::aarch64::*;
#[cfg(target_arch = "arm")]
use core::arch::arm::*;
"#;

85
library/stdarch/crates/intrinsic-test/src/arm/intrinsic.rs
View file

@ -1,7 +1,4 @@
use crate::common::argument::ArgumentList;
use crate::common::indentation::Indentation;
use crate::common::intrinsic::{Intrinsic, IntrinsicDefinition};
use crate::common::intrinsic_helpers::{IntrinsicType, IntrinsicTypeDefinition, Sign, TypeKind};
use crate::common::intrinsic_helpers::IntrinsicType;
use std::ops::{Deref, DerefMut};
#[derive(Debug, Clone, PartialEq)]
@ -23,83 +20,3 @@ impl DerefMut for ArmIntrinsicType {
&mut self.data
}
}
impl IntrinsicDefinition<ArmIntrinsicType> for Intrinsic<ArmIntrinsicType> {
fn arguments(&self) -> ArgumentList<ArmIntrinsicType> {
self.arguments.clone()
}
fn results(&self) -> ArmIntrinsicType {
self.results.clone()
}
fn name(&self) -> String {
self.name.clone()
}
/// Generates a std::cout for the intrinsics results that will match the
/// rust debug output format for the return type. The generated line assumes
/// there is an int i in scope which is the current pass number.
fn print_result_c(&self, indentation: Indentation, additional: &str) -> String {
let lanes = if self.results().num_vectors() > 1 {
(0..self.results().num_vectors())
.map(|vector| {
format!(
r#""{ty}(" << {lanes} << ")""#,
ty = self.results().c_single_vector_type(),
lanes = (0..self.results().num_lanes())
.map(move |idx| -> std::string::String {
format!(
"{cast}{lane_fn}(__return_value.val[{vector}], {lane})",
cast = self.results().c_promotion(),
lane_fn = self.results().get_lane_function(),
lane = idx,
vector = vector,
)
})
.collect::<Vec<_>>()
.join(r#" << ", " << "#)
)
})
.collect::<Vec<_>>()
.join(r#" << ", " << "#)
} else if self.results().num_lanes() > 1 {
(0..self.results().num_lanes())
.map(|idx| -> std::string::String {
format!(
"{cast}{lane_fn}(__return_value, {lane})",
cast = self.results().c_promotion(),
lane_fn = self.results().get_lane_function(),
lane = idx
)
})
.collect::<Vec<_>>()
.join(r#" << ", " << "#)
} else {
format!(
"{promote}cast<{cast}>(__return_value)",
cast = match self.results.kind() {
TypeKind::Float if self.results().inner_size() == 16 => "float16_t".to_string(),
TypeKind::Float if self.results().inner_size() == 32 => "float".to_string(),
TypeKind::Float if self.results().inner_size() == 64 => "double".to_string(),
TypeKind::Int(Sign::Signed) => format!("int{}_t", self.results().inner_size()),
TypeKind::Int(Sign::Unsigned) =>
format!("uint{}_t", self.results().inner_size()),
TypeKind::Poly => format!("poly{}_t", self.results().inner_size()),
ty => todo!("print_result_c - Unknown type: {:#?}", ty),
},
promote = self.results().c_promotion(),
)
};
format!(
r#"{indentation}std::cout << "Result {additional}-" << i+1 << ": {ty}" << std::fixed << std::setprecision(150) << {lanes} << "{close}" << std::endl;"#,
ty = if self.results().is_simd() {
format!("{}(", self.results().c_type())
} else {
String::from("")
},
close = if self.results.is_simd() { ")" } else { "" },
)
}
}

2
library/stdarch/crates/intrinsic-test/src/arm/json_parser.rs
View file

@ -113,7 +113,7 @@ fn json_to_intrinsic(
Ok(Intrinsic {
name,
arguments,
results: results,
results,
arch_tags: intr.architectures,
})
}

180
library/stdarch/crates/intrinsic-test/src/arm/mod.rs
View file

@ -5,20 +5,11 @@ mod intrinsic;
mod json_parser;
mod types;
use std::fs::{self, File};
use rayon::prelude::*;
use crate::common::SupportedArchitectureTest;
use crate::common::cli::ProcessedCli;
use crate::common::compare::compare_outputs;
use crate::common::gen_c::{write_main_cpp, write_mod_cpp};
use crate::common::gen_rust::{
compile_rust_programs, write_bin_cargo_toml, write_lib_cargo_toml, write_lib_rs, write_main_rs,
};
use crate::common::compile_c::CppCompilation;
use crate::common::intrinsic::Intrinsic;
use crate::common::intrinsic_helpers::TypeKind;
use crate::common::{SupportedArchitectureTest, chunk_info};
use config::{AARCH_CONFIGURATIONS, F16_FORMATTING_DEF, POLY128_OSTREAM_DEF, build_notices};
use intrinsic::ArmIntrinsicType;
use json_parser::get_neon_intrinsics;
@ -28,7 +19,30 @@ pub struct ArmArchitectureTest {
}
impl SupportedArchitectureTest for ArmArchitectureTest {
fn create(cli_options: ProcessedCli) -> Box<Self> {
type IntrinsicImpl = ArmIntrinsicType;
fn cli_options(&self) -> &ProcessedCli {
&self.cli_options
}
fn intrinsics(&self) -> &[Intrinsic<ArmIntrinsicType>] {
&self.intrinsics
}
const NOTICE: &str = config::NOTICE;
const PLATFORM_C_HEADERS: &[&str] = &["arm_neon.h", "arm_acle.h", "arm_fp16.h"];
const PLATFORM_C_DEFINITIONS: &str = config::POLY128_OSTREAM_DEF;
const PLATFORM_C_FORWARD_DECLARATIONS: &str = config::POLY128_OSTREAM_DECL;
const PLATFORM_RUST_DEFINITIONS: &str = config::F16_FORMATTING_DEF;
const PLATFORM_RUST_CFGS: &str = config::AARCH_CONFIGURATIONS;
fn cpp_compilation(&self) -> Option<CppCompilation> {
compile::build_cpp_compilation(&self.cli_options)
}
fn create(cli_options: ProcessedCli) -> Self {
let a32 = cli_options.target.contains("v7");
let mut intrinsics = get_neon_intrinsics(&cli_options.filename, &cli_options.target)
.expect("Error parsing input file");
@ -50,149 +64,9 @@ impl SupportedArchitectureTest for ArmArchitectureTest {
.collect::<Vec<_>>();
intrinsics.dedup();
Box::new(Self {
Self {
intrinsics,
cli_options,
})
}
fn build_c_file(&self) -> bool {
let c_target = "aarch64";
let platform_headers = &["arm_neon.h", "arm_acle.h", "arm_fp16.h"];
let (chunk_size, chunk_count) = chunk_info(self.intrinsics.len());
let cpp_compiler_wrapped = compile::build_cpp_compilation(&self.cli_options);
let notice = &build_notices("// ");
fs::create_dir_all("c_programs").unwrap();
self.intrinsics
.par_chunks(chunk_size)
.enumerate()
.map(|(i, chunk)| {
let c_filename = format!("c_programs/mod_{i}.cpp");
let mut file = File::create(&c_filename).unwrap();
write_mod_cpp(&mut file, notice, c_target, platform_headers, chunk).unwrap();
// compile this cpp file into a .o file.
//
// This is done because `cpp_compiler_wrapped` is None when
// the --generate-only flag is passed
if let Some(cpp_compiler) = cpp_compiler_wrapped.as_ref() {
let output = cpp_compiler
.compile_object_file(&format!("mod_{i}.cpp"), &format!("mod_{i}.o"))?;
assert!(output.status.success(), "{output:?}");
}
Ok(())
})
.collect::<Result<(), std::io::Error>>()
.unwrap();
let mut file = File::create("c_programs/main.cpp").unwrap();
write_main_cpp(
&mut file,
c_target,
POLY128_OSTREAM_DEF,
self.intrinsics.iter().map(|i| i.name.as_str()),
)
.unwrap();
// This is done because `cpp_compiler_wrapped` is None when
// the --generate-only flag is passed
if let Some(cpp_compiler) = cpp_compiler_wrapped.as_ref() {
// compile this cpp file into a .o file
info!("compiling main.cpp");
let output = cpp_compiler
.compile_object_file("main.cpp", "intrinsic-test-programs.o")
.unwrap();
assert!(output.status.success(), "{output:?}");
let object_files = (0..chunk_count)
.map(|i| format!("mod_{i}.o"))
.chain(["intrinsic-test-programs.o".to_owned()]);
let output = cpp_compiler
.link_executable(object_files, "intrinsic-test-programs")
.unwrap();
assert!(output.status.success(), "{output:?}");
}
true
}
fn build_rust_file(&self) -> bool {
std::fs::create_dir_all("rust_programs/src").unwrap();
let architecture = if self.cli_options.target.contains("v7") {
"arm"
} else {
"aarch64"
};
let (chunk_size, chunk_count) = chunk_info(self.intrinsics.len());
let mut cargo = File::create("rust_programs/Cargo.toml").unwrap();
write_bin_cargo_toml(&mut cargo, chunk_count).unwrap();
let mut main_rs = File::create("rust_programs/src/main.rs").unwrap();
write_main_rs(
&mut main_rs,
chunk_count,
AARCH_CONFIGURATIONS,
"",
self.intrinsics.iter().map(|i| i.name.as_str()),
)
.unwrap();
let target = &self.cli_options.target;
let toolchain = self.cli_options.toolchain.as_deref();
let linker = self.cli_options.linker.as_deref();
let notice = &build_notices("// ");
self.intrinsics
.par_chunks(chunk_size)
.enumerate()
.map(|(i, chunk)| {
std::fs::create_dir_all(format!("rust_programs/mod_{i}/src"))?;
let rust_filename = format!("rust_programs/mod_{i}/src/lib.rs");
trace!("generating `{rust_filename}`");
let mut file = File::create(rust_filename)?;
let cfg = AARCH_CONFIGURATIONS;
let definitions = F16_FORMATTING_DEF;
write_lib_rs(&mut file, architecture, notice, cfg, definitions, chunk)?;
let toml_filename = format!("rust_programs/mod_{i}/Cargo.toml");
trace!("generating `{toml_filename}`");
let mut file = File::create(toml_filename).unwrap();
write_lib_cargo_toml(&mut file, &format!("mod_{i}"))?;
Ok(())
})
.collect::<Result<(), std::io::Error>>()
.unwrap();
compile_rust_programs(toolchain, target, linker)
}
fn compare_outputs(&self) -> bool {
if self.cli_options.toolchain.is_some() {
let intrinsics_name_list = self
.intrinsics
.iter()
.map(|i| i.name.clone())
.collect::<Vec<_>>();
compare_outputs(
&intrinsics_name_list,
&self.cli_options.runner,
&self.cli_options.target,
)
} else {
true
}
}
}

66
library/stdarch/crates/intrinsic-test/src/arm/types.rs
View file

@ -1,5 +1,6 @@
use super::intrinsic::ArmIntrinsicType;
use crate::common::cli::Language;
use crate::common::indentation::Indentation;
use crate::common::intrinsic_helpers::{IntrinsicType, IntrinsicTypeDefinition, Sign, TypeKind};
impl IntrinsicTypeDefinition for ArmIntrinsicType {
@ -98,6 +99,71 @@ impl IntrinsicTypeDefinition for ArmIntrinsicType {
todo!("get_lane_function IntrinsicType: {:#?}", self)
}
}
/// Generates a std::cout for the intrinsics results that will match the
/// rust debug output format for the return type. The generated line assumes
/// there is an int i in scope which is the current pass number.
fn print_result_c(&self, indentation: Indentation, additional: &str) -> String {
let lanes = if self.num_vectors() > 1 {
(0..self.num_vectors())
.map(|vector| {
format!(
r#""{ty}(" << {lanes} << ")""#,
ty = self.c_single_vector_type(),
lanes = (0..self.num_lanes())
.map(move |idx| -> std::string::String {
format!(
"{cast}{lane_fn}(__return_value.val[{vector}], {lane})",
cast = self.c_promotion(),
lane_fn = self.get_lane_function(),
lane = idx,
vector = vector,
)
})
.collect::<Vec<_>>()
.join(r#" << ", " << "#)
)
})
.collect::<Vec<_>>()
.join(r#" << ", " << "#)
} else if self.num_lanes() > 1 {
(0..self.num_lanes())
.map(|idx| -> std::string::String {
format!(
"{cast}{lane_fn}(__return_value, {lane})",
cast = self.c_promotion(),
lane_fn = self.get_lane_function(),
lane = idx
)
})
.collect::<Vec<_>>()
.join(r#" << ", " << "#)
} else {
format!(
"{promote}cast<{cast}>(__return_value)",
cast = match self.kind() {
TypeKind::Float if self.inner_size() == 16 => "float16_t".to_string(),
TypeKind::Float if self.inner_size() == 32 => "float".to_string(),
TypeKind::Float if self.inner_size() == 64 => "double".to_string(),
TypeKind::Int(Sign::Signed) => format!("int{}_t", self.inner_size()),
TypeKind::Int(Sign::Unsigned) => format!("uint{}_t", self.inner_size()),
TypeKind::Poly => format!("poly{}_t", self.inner_size()),
ty => todo!("print_result_c - Unknown type: {:#?}", ty),
},
promote = self.c_promotion(),
)
};
format!(
r#"{indentation}std::cout << "Result {additional}-" << i+1 << ": {ty}" << std::fixed << std::setprecision(150) << {lanes} << "{close}" << std::endl;"#,
ty = if self.is_simd() {
format!("{}(", self.c_type())
} else {
String::from("")
},
close = if self.is_simd() { ")" } else { "" },
)
}
}
impl ArmIntrinsicType {

38
library/stdarch/crates/intrinsic-test/src/common/gen_c.rs
View file

@ -1,6 +1,7 @@
use crate::common::intrinsic::Intrinsic;
use super::argument::Argument;
use super::indentation::Indentation;
use super::intrinsic::IntrinsicDefinition;
use super::intrinsic_helpers::IntrinsicTypeDefinition;
// The number of times each intrinsic will be called.
@ -8,7 +9,7 @@ const PASSES: u32 = 20;
pub fn generate_c_test_loop<T: IntrinsicTypeDefinition + Sized>(
w: &mut impl std::io::Write,
intrinsic: &dyn IntrinsicDefinition<T>,
intrinsic: &Intrinsic<T>,
indentation: Indentation,
additional: &str,
passes: u32,
@ -21,16 +22,18 @@ pub fn generate_c_test_loop<T: IntrinsicTypeDefinition + Sized>(
{body_indentation}auto __return_value = {intrinsic_call}({args});\n\
{print_result}\n\
{indentation}}}",
loaded_args = intrinsic.arguments().load_values_c(body_indentation),
intrinsic_call = intrinsic.name(),
args = intrinsic.arguments().as_call_param_c(),
print_result = intrinsic.print_result_c(body_indentation, additional)
loaded_args = intrinsic.arguments.load_values_c(body_indentation),
intrinsic_call = intrinsic.name,
args = intrinsic.arguments.as_call_param_c(),
print_result = intrinsic
.results
.print_result_c(body_indentation, additional)
)
}
pub fn generate_c_constraint_blocks<'a, T: IntrinsicTypeDefinition + 'a>(
w: &mut impl std::io::Write,
intrinsic: &dyn IntrinsicDefinition<T>,
intrinsic: &Intrinsic<T>,
indentation: Indentation,
constraints: &mut (impl Iterator<Item = &'a Argument<T>> + Clone),
name: String,
@ -63,14 +66,14 @@ pub fn generate_c_constraint_blocks<'a, T: IntrinsicTypeDefinition + 'a>(
// Compiles C test programs using specified compiler
pub fn create_c_test_function<T: IntrinsicTypeDefinition>(
w: &mut impl std::io::Write,
intrinsic: &dyn IntrinsicDefinition<T>,
intrinsic: &Intrinsic<T>,
) -> std::io::Result<()> {
let indentation = Indentation::default();
writeln!(w, "int run_{}() {{", intrinsic.name())?;
writeln!(w, "int run_{}() {{", intrinsic.name)?;
// Define the arrays of arguments.
let arguments = intrinsic.arguments();
let arguments = &intrinsic.arguments;
arguments.gen_arglists_c(w, indentation.nested(), PASSES)?;
generate_c_constraint_blocks(
@ -90,9 +93,9 @@ pub fn create_c_test_function<T: IntrinsicTypeDefinition>(
pub fn write_mod_cpp<T: IntrinsicTypeDefinition>(
w: &mut impl std::io::Write,
notice: &str,
architecture: &str,
platform_headers: &[&str],
intrinsics: &[impl IntrinsicDefinition<T>],
forward_declarations: &str,
intrinsics: &[Intrinsic<T>],
) -> std::io::Result<()> {
write!(w, "{notice}")?;
@ -122,12 +125,7 @@ std::ostream& operator<<(std::ostream& os, float16_t value);
"#
)?;
writeln!(w, "#ifdef __{architecture}__")?;
writeln!(
w,
"std::ostream& operator<<(std::ostream& os, poly128_t value);"
)?;
writeln!(w, "#endif")?;
writeln!(w, "{}", forward_declarations)?;
for intrinsic in intrinsics {
create_c_test_function(w, intrinsic)?;
@ -138,7 +136,6 @@ std::ostream& operator<<(std::ostream& os, float16_t value);
pub fn write_main_cpp<'a>(
w: &mut impl std::io::Write,
architecture: &str,
arch_specific_definitions: &str,
intrinsics: impl Iterator<Item = &'a str> + Clone,
) -> std::io::Result<()> {
@ -167,9 +164,8 @@ std::ostream& operator<<(std::ostream& os, float16_t value) {{
"#
)?;
writeln!(w, "#ifdef __{architecture}__")?;
// NOTE: It's assumed that this value contains the required `ifdef`s.
writeln!(w, "{arch_specific_definitions }")?;
writeln!(w, "#endif")?;
for intrinsic in intrinsics.clone() {
writeln!(w, "extern int run_{intrinsic}(void);")?;

29
library/stdarch/crates/intrinsic-test/src/common/gen_rust.rs
View file

@ -1,8 +1,10 @@
use itertools::Itertools;
use std::process::Command;
use crate::common::intrinsic::Intrinsic;
use super::indentation::Indentation;
use super::intrinsic::{IntrinsicDefinition, format_f16_return_value};
use super::intrinsic::format_f16_return_value;
use super::intrinsic_helpers::IntrinsicTypeDefinition;
// The number of times each intrinsic will be called.
@ -96,11 +98,10 @@ pub fn write_main_rs<'a>(
pub fn write_lib_rs<T: IntrinsicTypeDefinition>(
w: &mut impl std::io::Write,
architecture: &str,
notice: &str,
cfg: &str,
definitions: &str,
intrinsics: &[impl IntrinsicDefinition<T>],
intrinsics: &[Intrinsic<T>],
) -> std::io::Result<()> {
write!(w, "{notice}")?;
@ -115,8 +116,6 @@ pub fn write_lib_rs<T: IntrinsicTypeDefinition>(
writeln!(w, "{cfg}")?;
writeln!(w, "use core_arch::arch::{architecture}::*;")?;
writeln!(w, "{definitions}")?;
for intrinsic in intrinsics {
@ -189,16 +188,16 @@ pub fn compile_rust_programs(toolchain: Option<&str>, target: &str, linker: Opti
pub fn generate_rust_test_loop<T: IntrinsicTypeDefinition>(
w: &mut impl std::io::Write,
intrinsic: &dyn IntrinsicDefinition<T>,
intrinsic: &Intrinsic<T>,
indentation: Indentation,
specializations: &[Vec<u8>],
passes: u32,
) -> std::io::Result<()> {
let intrinsic_name = intrinsic.name();
let intrinsic_name = &intrinsic.name;
// Each function (and each specialization) has its own type. Erase that type with a cast.
let mut coerce = String::from("unsafe fn(");
for _ in intrinsic.arguments().iter().filter(|a| !a.has_constraint()) {
for _ in intrinsic.arguments.iter().filter(|a| !a.has_constraint()) {
coerce += "_, ";
}
coerce += ") -> _";
@ -248,13 +247,13 @@ pub fn generate_rust_test_loop<T: IntrinsicTypeDefinition>(
}}\n\
}}\n\
}}",
loaded_args = intrinsic.arguments().load_values_rust(indentation3),
args = intrinsic.arguments().as_call_param_rust(),
loaded_args = intrinsic.arguments.load_values_rust(indentation3),
args = intrinsic.arguments.as_call_param_rust(),
)
}
/// Generate the specializations (unique sequences of const-generic arguments) for this intrinsic.
fn generate_rust_specializations<'a>(
fn generate_rust_specializations(
constraints: &mut impl Iterator<Item = impl Iterator<Item = i64>>,
) -> Vec<Vec<u8>> {
let mut specializations = vec![vec![]];
@ -277,15 +276,15 @@ fn generate_rust_specializations<'a>(
// Top-level function to create complete test program
pub fn create_rust_test_module<T: IntrinsicTypeDefinition>(
w: &mut impl std::io::Write,
intrinsic: &dyn IntrinsicDefinition<T>,
intrinsic: &Intrinsic<T>,
) -> std::io::Result<()> {
trace!("generating `{}`", intrinsic.name());
trace!("generating `{}`", intrinsic.name);
let indentation = Indentation::default();
writeln!(w, "pub fn run_{}() {{", intrinsic.name())?;
writeln!(w, "pub fn run_{}() {{", intrinsic.name)?;
// Define the arrays of arguments.
let arguments = intrinsic.arguments();
let arguments = &intrinsic.arguments;
arguments.gen_arglists_rust(w, indentation.nested(), PASSES)?;
// Define any const generics as `const` items, then generate the actual test loop.

25
library/stdarch/crates/intrinsic-test/src/common/intrinsic.rs
View file

@ -1,5 +1,4 @@
use super::argument::ArgumentList;
use super::indentation::Indentation;
use super::intrinsic_helpers::{IntrinsicTypeDefinition, TypeKind};
/// An intrinsic
@ -18,32 +17,14 @@ pub struct Intrinsic<T: IntrinsicTypeDefinition> {
pub arch_tags: Vec<String>,
}
pub trait IntrinsicDefinition<T>
where
T: IntrinsicTypeDefinition,
{
fn arguments(&self) -> ArgumentList<T>;
fn results(&self) -> T;
fn name(&self) -> String;
/// Generates a std::cout for the intrinsics results that will match the
/// rust debug output format for the return type. The generated line assumes
/// there is an int i in scope which is the current pass number.
fn print_result_c(&self, _indentation: Indentation, _additional: &str) -> String;
}
pub fn format_f16_return_value<T: IntrinsicTypeDefinition>(
intrinsic: &dyn IntrinsicDefinition<T>,
) -> String {
pub fn format_f16_return_value<T: IntrinsicTypeDefinition>(intrinsic: &Intrinsic<T>) -> String {
// the `intrinsic-test` crate compares the output of C and Rust intrinsics. Currently, It uses
// a string representation of the output value to compare. In C, f16 values are currently printed
// as hexadecimal integers. Since https://github.com/rust-lang/rust/pull/127013, rust does print
// them as decimal floating point values. To keep the intrinsics tests working, for now, format
// vectors containing f16 values like C prints them.
let return_value = match intrinsic.results().kind() {
TypeKind::Float if intrinsic.results().inner_size() == 16 => "debug_f16(__return_value)",
let return_value = match intrinsic.results.kind() {
TypeKind::Float if intrinsic.results.inner_size() == 16 => "debug_f16(__return_value)",
_ => "format_args!(\"{__return_value:.150?}\")",
};

5
library/stdarch/crates/intrinsic-test/src/common/intrinsic_helpers.rs
View file

@ -325,4 +325,9 @@ pub trait IntrinsicTypeDefinition: Deref<Target = IntrinsicType> {
/// can be directly defined in `impl` blocks
fn c_single_vector_type(&self) -> String;
/// Generates a std::cout for the intrinsics results that will match the
/// rust debug output format for the return type. The generated line assumes
/// there is an int i in scope which is the current pass number.
fn print_result_c(&self, indentation: Indentation, additional: &str) -> String;
}

177
library/stdarch/crates/intrinsic-test/src/common/mod.rs
View file

@ -1,5 +1,20 @@
use std::fs::File;
use rayon::prelude::*;
use cli::ProcessedCli;
use crate::common::{
compile_c::CppCompilation,
gen_c::{write_main_cpp, write_mod_cpp},
gen_rust::{
compile_rust_programs, write_bin_cargo_toml, write_lib_cargo_toml, write_lib_rs,
write_main_rs,
},
intrinsic::Intrinsic,
intrinsic_helpers::IntrinsicTypeDefinition,
};
pub mod argument;
pub mod cli;
pub mod compare;
@ -15,12 +30,162 @@ pub mod values;
/// Architectures must support this trait
/// to be successfully tested.
pub trait SupportedArchitectureTest {
fn create(cli_options: ProcessedCli) -> Box<Self>
where
Self: Sized;
fn build_c_file(&self) -> bool;
fn build_rust_file(&self) -> bool;
fn compare_outputs(&self) -> bool;
type IntrinsicImpl: IntrinsicTypeDefinition + Sync;
fn cli_options(&self) -> &ProcessedCli;
fn intrinsics(&self) -> &[Intrinsic<Self::IntrinsicImpl>];
fn create(cli_options: ProcessedCli) -> Self;
const NOTICE: &str;
const PLATFORM_C_HEADERS: &[&str];
const PLATFORM_C_DEFINITIONS: &str;
const PLATFORM_C_FORWARD_DECLARATIONS: &str;
const PLATFORM_RUST_CFGS: &str;
const PLATFORM_RUST_DEFINITIONS: &str;
fn cpp_compilation(&self) -> Option<CppCompilation>;
fn build_c_file(&self) -> bool {
let (chunk_size, chunk_count) = chunk_info(self.intrinsics().len());
let cpp_compiler_wrapped = self.cpp_compilation();
std::fs::create_dir_all("c_programs").unwrap();
self.intrinsics()
.par_chunks(chunk_size)
.enumerate()
.map(|(i, chunk)| {
let c_filename = format!("c_programs/mod_{i}.cpp");
let mut file = File::create(&c_filename).unwrap();
write_mod_cpp(
&mut file,
Self::NOTICE,
Self::PLATFORM_C_HEADERS,
Self::PLATFORM_C_FORWARD_DECLARATIONS,
chunk,
)
.unwrap();
// compile this cpp file into a .o file.
//
// This is done because `cpp_compiler_wrapped` is None when
// the --generate-only flag is passed
if let Some(cpp_compiler) = cpp_compiler_wrapped.as_ref() {
let output = cpp_compiler
.compile_object_file(&format!("mod_{i}.cpp"), &format!("mod_{i}.o"))?;
assert!(output.status.success(), "{output:?}");
}
Ok(())
})
.collect::<Result<(), std::io::Error>>()
.unwrap();
let mut file = File::create("c_programs/main.cpp").unwrap();
write_main_cpp(
&mut file,
Self::PLATFORM_C_DEFINITIONS,
self.intrinsics().iter().map(|i| i.name.as_str()),
)
.unwrap();
// This is done because `cpp_compiler_wrapped` is None when
// the --generate-only flag is passed
if let Some(cpp_compiler) = cpp_compiler_wrapped.as_ref() {
// compile this cpp file into a .o file
info!("compiling main.cpp");
let output = cpp_compiler
.compile_object_file("main.cpp", "intrinsic-test-programs.o")
.unwrap();
assert!(output.status.success(), "{output:?}");
let object_files = (0..chunk_count)
.map(|i| format!("mod_{i}.o"))
.chain(["intrinsic-test-programs.o".to_owned()]);
let output = cpp_compiler
.link_executable(object_files, "intrinsic-test-programs")
.unwrap();
assert!(output.status.success(), "{output:?}");
}
true
}
fn build_rust_file(&self) -> bool {
std::fs::create_dir_all("rust_programs/src").unwrap();
let (chunk_size, chunk_count) = chunk_info(self.intrinsics().len());
let mut cargo = File::create("rust_programs/Cargo.toml").unwrap();
write_bin_cargo_toml(&mut cargo, chunk_count).unwrap();
let mut main_rs = File::create("rust_programs/src/main.rs").unwrap();
write_main_rs(
&mut main_rs,
chunk_count,
Self::PLATFORM_RUST_CFGS,
"",
self.intrinsics().iter().map(|i| i.name.as_str()),
)
.unwrap();
let target = &self.cli_options().target;
let toolchain = self.cli_options().toolchain.as_deref();
let linker = self.cli_options().linker.as_deref();
self.intrinsics()
.par_chunks(chunk_size)
.enumerate()
.map(|(i, chunk)| {
std::fs::create_dir_all(format!("rust_programs/mod_{i}/src"))?;
let rust_filename = format!("rust_programs/mod_{i}/src/lib.rs");
trace!("generating `{rust_filename}`");
let mut file = File::create(rust_filename)?;
write_lib_rs(
&mut file,
Self::NOTICE,
Self::PLATFORM_RUST_CFGS,
Self::PLATFORM_RUST_DEFINITIONS,
chunk,
)?;
let toml_filename = format!("rust_programs/mod_{i}/Cargo.toml");
trace!("generating `{toml_filename}`");
let mut file = File::create(toml_filename).unwrap();
write_lib_cargo_toml(&mut file, &format!("mod_{i}"))?;
Ok(())
})
.collect::<Result<(), std::io::Error>>()
.unwrap();
compile_rust_programs(toolchain, target, linker)
}
fn compare_outputs(&self) -> bool {
if self.cli_options().toolchain.is_some() {
let intrinsics_name_list = self
.intrinsics()
.iter()
.map(|i| i.name.clone())
.collect::<Vec<_>>();
compare::compare_outputs(
&intrinsics_name_list,
&self.cli_options().runner,
&self.cli_options().target,
)
} else {
true
}
}
}
pub fn chunk_info(intrinsic_count: usize) -> (usize, usize) {

21
library/stdarch/crates/intrinsic-test/src/main.rs
View file

@ -13,23 +13,16 @@ fn main() {
let args: Cli = clap::Parser::parse();
let processed_cli_options = ProcessedCli::new(args);
let test_environment_result: Option<Box<dyn SupportedArchitectureTest>> =
match processed_cli_options.target.as_str() {
"aarch64-unknown-linux-gnu"
| "armv7-unknown-linux-gnueabihf"
| "aarch64_be-unknown-linux-gnu" => {
Some(ArmArchitectureTest::create(processed_cli_options))
}
match processed_cli_options.target.as_str() {
"aarch64-unknown-linux-gnu"
| "armv7-unknown-linux-gnueabihf"
| "aarch64_be-unknown-linux-gnu" => run(ArmArchitectureTest::create(processed_cli_options)),
_ => None,
};
if test_environment_result.is_none() {
std::process::exit(0);
_ => std::process::exit(0),
}
}
let test_environment = test_environment_result.unwrap();
fn run(test_environment: impl SupportedArchitectureTest) {
info!("building C binaries");
if !test_environment.build_c_file() {
std::process::exit(2);

146
library/stdarch/crates/stdarch-gen-arm/spec/neon/aarch64.spec.yml
View file

File diff suppressed because it is too large Load diff

167
library/stdarch/crates/stdarch-gen-arm/spec/neon/arm_shared.spec.yml
View file

@ -37,6 +37,10 @@ target-is-arm: &target-is-arm
target-not-arm: &target-not-arm
FnCall: [cfg, [{ FnCall: [not, ['target_arch = "arm"']]}]]
# #[cfg(not(target_arch = "arm64ec"))]
target-not-arm64ec: &target-not-arm64ec
FnCall: [cfg, [{ FnCall: [not, ['target_arch = "arm64ec"']]}]]
not-arm: &not-arm
FnCall: [not, ['target_arch = "arm"']]
@ -278,6 +282,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fabd]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -396,6 +401,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fcmeq]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, uint16x4_t]
@ -457,6 +463,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fabs]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -474,6 +481,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fabs]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- ['h_f16', 'f16']
@ -555,6 +563,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fcmgt]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, uint16x4_t]
@ -573,6 +582,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fcmgt]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, uint16x4_t, f16x4, 'f16x4::new(0.0, 0.0, 0.0, 0.0)']
@ -651,6 +661,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fcmge]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, uint16x4_t]
@ -668,6 +679,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fcmle]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, uint16x4_t, f16x4, 'f16x4::new(0.0, 0.0, 0.0, 0.0)']
@ -849,6 +861,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [facgt]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, uint16x4_t]
@ -895,6 +908,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [facge]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, uint16x4_t]
@ -935,6 +949,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [facgt]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, uint16x4_t]
@ -970,6 +985,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [facge]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, uint16x4_t]
@ -1004,6 +1020,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [scvtf]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [int16x4_t, float16x4_t]
@ -1038,6 +1055,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ucvtf]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [uint16x4_t, float16x4_t]
@ -1109,6 +1127,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ['1']]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const N: i32']
safety: safe
types:
@ -1140,6 +1159,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ['1']]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const N: i32']
safety: safe
types:
@ -1171,6 +1191,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ['1']]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const N: i32']
safety: safe
types:
@ -1229,6 +1250,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ['1']]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const N: i32']
safety: safe
types:
@ -1482,6 +1504,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ['1']]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const N: i32']
safety: safe
types:
@ -1501,6 +1524,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [dup]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, f16, 'float16x4', '_n_']
@ -1519,6 +1543,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ['1']]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const N: i32']
safety: safe
types:
@ -1740,6 +1765,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ['2']]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const N: i32']
safety: safe
types:
@ -1758,6 +1784,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ['2']]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const N: i32']
safety: safe
types:
@ -2207,6 +2234,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fneg]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, 'f16']
@ -2262,13 +2290,10 @@ intrinsics:
- [uint64x1_t, u64, i64]
- [uint64x2_t, u64, i64]
compose:
- LLVMLink:
name: "uqsub.{neon_type[0]}"
links:
- link: "llvm.aarch64.neon.uqsub.v{neon_type[0].lane}{type[2]}"
arch: aarch64,arm64ec
- link: "llvm.usub.sat.v{neon_type[0].lane}{type[2]}"
arch: arm
- FnCall:
- simd_saturating_sub
- - a
- b
- name: "vqsub{neon_type[0].no}"
doc: Saturating subtract
@ -2291,13 +2316,10 @@ intrinsics:
- [int64x1_t, s64, i64]
- [int64x2_t, s64, i64]
compose:
- LLVMLink:
name: "sqsub.{neon_type[0]}"
links:
- link: "llvm.aarch64.neon.sqsub.v{neon_type[0].lane}{type[2]}"
arch: aarch64,arm64ec
- link: "llvm.ssub.sat.v{neon_type[0].lane}{type[2]}"
arch: arm
- FnCall:
- simd_saturating_sub
- - a
- b
- name: "vhadd{neon_type.no}"
doc: Halving add
@ -2464,9 +2486,7 @@ intrinsics:
name: "llvm.frinn.{neon_type}"
links:
- link: "llvm.roundeven.{neon_type}"
arch: aarch64,arm64ec
- link: "llvm.arm.neon.vrintn.{neon_type}"
arch: arm
arch: aarch64,arm64ec,arm
- name: "vrndn{neon_type.no}"
doc: "Floating-point round to integral, to nearest with ties to even"
@ -2478,6 +2498,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [frintn]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -2487,9 +2508,7 @@ intrinsics:
name: "llvm.frinn.{neon_type}"
links:
- link: "llvm.roundeven.{neon_type}"
arch: aarch64,arm64ec
- link: "llvm.arm.neon.vrintn.{neon_type}"
arch: arm
arch: aarch64,arm64ec,arm
- name: "vqadd{neon_type.no}"
doc: Saturating add
@ -2512,13 +2531,10 @@ intrinsics:
- uint64x1_t
- uint64x2_t
compose:
- LLVMLink:
name: "uqadd.{neon_type}"
links:
- link: "llvm.aarch64.neon.uqadd.{neon_type}"
arch: aarch64,arm64ec
- link: "llvm.uadd.sat.{neon_type}"
arch: arm
- FnCall:
- simd_saturating_add
- - a
- b
- name: "vqadd{neon_type.no}"
doc: Saturating add
@ -2541,13 +2557,10 @@ intrinsics:
- int64x1_t
- int64x2_t
compose:
- LLVMLink:
name: "sqadd.{neon_type}"
links:
- link: "llvm.aarch64.neon.sqadd.{neon_type}"
arch: aarch64,arm64ec
- link: "llvm.sadd.sat.{neon_type}"
arch: arm
- FnCall:
- simd_saturating_add
- - a
- b
- name: "vld1{neon_type[1].no}"
doc: "Load multiple single-element structures to one, two, three, or four registers"
@ -2743,6 +2756,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld1]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -2773,6 +2787,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ["2"]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const LANE: i32']
safety:
unsafe: [neon]
@ -2793,6 +2808,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld1r]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -3385,6 +3401,7 @@ intrinsics:
- FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld2]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -3413,6 +3430,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld2]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -3440,6 +3458,7 @@ intrinsics:
- *neon-fp16
- FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld2]]}]]
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -3469,6 +3488,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld2r]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -3498,6 +3518,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ["2"]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs:
- "const LANE: i32"
safety:
@ -3540,6 +3561,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ["2"]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs:
- "const LANE: i32"
safety:
@ -3580,6 +3602,7 @@ intrinsics:
- FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld3]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -3608,6 +3631,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld3]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -3635,6 +3659,7 @@ intrinsics:
- FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld3]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -3664,6 +3689,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld3r]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -3693,6 +3719,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ["2"]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs:
- "const LANE: i32"
safety:
@ -3737,6 +3764,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ["2"]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs:
- "const LANE: i32"
safety:
@ -4718,6 +4746,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ["2"]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
types:
- ['*mut f16', float16x4_t, '2']
- ['*mut f16', float16x8_t, '3']
@ -4955,6 +4984,7 @@ intrinsics:
- *neon-v7
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
assert_instr: [vst1]
types:
- [f16, float16x4x4_t, float16x4_t]
@ -5098,6 +5128,7 @@ intrinsics:
- *target-not-arm
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
assert_instr: [st2]
safety:
unsafe: [neon]
@ -5188,6 +5219,7 @@ intrinsics:
- FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st2, 'LANE = 0']]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const LANE: i32']
safety:
unsafe: [neon]
@ -5279,6 +5311,7 @@ intrinsics:
- *neon-v7
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
assert_instr: [vst2]
safety:
unsafe: [neon]
@ -5345,6 +5378,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ['2']]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const LANE: i32']
safety:
unsafe: [neon]
@ -5555,6 +5589,7 @@ intrinsics:
- *neon-v7
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
assert_instr: [vst3]
safety:
unsafe: [neon]
@ -5623,6 +5658,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ['2']]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const LANE: i32']
safety:
unsafe: [neon]
@ -5683,6 +5719,7 @@ intrinsics:
- *target-not-arm
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
assert_instr: [st3]
safety:
unsafe: [neon]
@ -5747,6 +5784,7 @@ intrinsics:
- FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st3, 'LANE = 0']]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const LANE: i32']
safety:
unsafe: [neon]
@ -5960,6 +5998,7 @@ intrinsics:
- *neon-v7
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
assert_instr: [vst4]
safety:
unsafe: [neon]
@ -6029,6 +6068,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ['2']]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const LANE: i32']
safety:
unsafe: [neon]
@ -6091,6 +6131,7 @@ intrinsics:
- *target-not-arm
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
assert_instr: [st4]
safety:
unsafe: [neon]
@ -6157,6 +6198,7 @@ intrinsics:
- FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st4, 'LANE = 0']]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const LANE: i32']
safety:
unsafe: [neon]
@ -6317,6 +6359,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmul]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [f16, float16x4_t]
@ -6366,6 +6409,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ['2']]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ["const LANE: i32"]
safety: safe
types:
@ -6569,6 +6613,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmla]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -6678,6 +6723,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fsub]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- ['f16', float16x4_t]
@ -6696,6 +6742,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fadd]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -6716,6 +6763,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fadd]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- ['h_f16', 'f16']
@ -7194,6 +7242,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmax]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -7236,6 +7285,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmaxnm]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -7254,6 +7304,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fminnm]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -7340,6 +7391,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmin]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -7404,6 +7456,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [faddp]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -8247,6 +8300,7 @@ intrinsics:
- FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vrsqrts]]}]]
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [frsqrts]]}]]
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -8295,6 +8349,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [frecpe]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -8343,6 +8398,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [frecps]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -8424,6 +8480,7 @@ intrinsics:
- [poly16x8_t, p128]
- [int8x16_t, p128]
- [uint8x16_t, p128]
big_endian_inverse: false
compose:
- FnCall: [transmute, [a]]
@ -8661,6 +8718,7 @@ intrinsics:
- [poly8x16_t, float32x4_t]
- [poly16x8_t, float32x4_t]
- [p128, float32x4_t]
big_endian_inverse: false
compose:
- FnCall: [transmute, [a]]
@ -8675,6 +8733,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
# non-q
@ -8723,6 +8782,7 @@ intrinsics:
- [float16x8_t, uint16x8_t]
- [float16x8_t, uint32x4_t]
- [float16x8_t, uint64x2_t]
big_endian_inverse: false
compose:
- FnCall: [transmute, [a]]
@ -8737,6 +8797,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [poly64x1_t, float16x4_t]
@ -8746,6 +8807,7 @@ intrinsics:
- [poly128_t, float16x8_t]
- [float16x8_t, poly128_t]
- [float16x8_t, poly64x2_t]
big_endian_inverse: false
compose:
- FnCall: [transmute, [a]]
@ -8759,6 +8821,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [rev64]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, "[3, 2, 1, 0]"]
@ -9074,6 +9137,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- ["u64", float16x4_t]
@ -9147,6 +9211,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ['2']]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const LANE: i32']
safety: safe
types:
@ -9578,6 +9643,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [trn2]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, float16x4x2_t, '[0, 4, 2, 6]', '[1, 5, 3, 7]']
@ -9733,6 +9799,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [zip2]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, float16x4x2_t, '[0, 4, 1, 5]', '[2, 6, 3, 7]']
@ -9803,6 +9870,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [uzp2]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, float16x4x2_t, '[0, 2, 4, 6]', '[1, 3, 5, 7]']
@ -10050,6 +10118,7 @@ intrinsics:
- FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vst1]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -10077,6 +10146,7 @@ intrinsics:
- FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -10103,6 +10173,7 @@ intrinsics:
- FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [vst1]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -10179,6 +10250,7 @@ intrinsics:
- FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -10233,6 +10305,7 @@ intrinsics:
- FnCall: [cfg_attr, [test, {FnCall: [assert_instr, [st1]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -10376,6 +10449,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fcmge]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, uint16x4_t]
@ -10394,6 +10468,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fcmge]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, uint16x4_t, f16x4, 'f16x4::new(0.0, 0.0, 0.0, 0.0)']
@ -10633,6 +10708,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fcvtzu]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, uint16x4_t]
@ -10652,6 +10728,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fcvtn]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float32x4_t, float16x4_t]
@ -10668,6 +10745,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fcvtl]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, float32x4_t]
@ -11029,6 +11107,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmul]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, "f16"]
@ -11141,6 +11220,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fcmgt]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, uint16x4_t]
@ -11159,6 +11239,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fcmlt]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, uint16x4_t, f16x4, 'f16x4::new(0.0, 0.0, 0.0, 0.0)']
@ -11271,6 +11352,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fmls]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -11386,6 +11468,7 @@ intrinsics:
- FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vrsqrte]]}]]
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [frsqrte]]}]]
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- float16x4_t
@ -11499,6 +11582,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [fcvtzs]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, int16x4_t]
@ -11748,6 +11832,7 @@ intrinsics:
- FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [nop]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -11834,6 +11919,7 @@ intrinsics:
- FnCall: [target_feature, ['enable = "{type[3]}"']]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
- FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['{type[2]}']]}]]
types:
- ['*const f16', float16x4_t, '"vld1.16"', 'neon,v7', 'crate::mem::align_of::<f16>() as i32', '_v4f16']
@ -12117,6 +12203,7 @@ intrinsics:
- FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld4]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -12145,6 +12232,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld4]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -12172,6 +12260,7 @@ intrinsics:
- FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, [vld4]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -12201,6 +12290,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [ld4r]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety:
unsafe: [neon]
types:
@ -12230,6 +12320,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ["2"]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs:
- "const LANE: i32"
safety:
@ -12276,6 +12367,7 @@ intrinsics:
- FnCall: [rustc_legacy_const_generics, ["2"]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
static_defs:
- "const LANE: i32"
safety:
@ -13674,6 +13766,7 @@ intrinsics:
- *neon-v7
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
- FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vst1.{type[4]}"']]}]]
types:
- ['_v4f16', '* const i8', float16x4_t, i32, '16']
@ -13738,6 +13831,7 @@ intrinsics:
- *neon-v7
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
- FnCall: [cfg_attr, [*test-is-arm, {FnCall: [assert_instr, ['"vst1.{type[2]}"']]}]]
types:
- ['*mut f16', float16x4_t, '16', 'transmute(a)', 'crate::mem::align_of::<f16>() as i32', '_v4f16']
@ -13918,6 +14012,7 @@ intrinsics:
- *neon-v7
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
assert_instr: [nop]
safety: safe
types:
@ -13933,6 +14028,7 @@ intrinsics:
- *neon-v7
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
assert_instr: [nop]
safety: safe
types:
@ -13952,6 +14048,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [nop, 'LANE = 0']]}]]
- FnCall: [rustc_legacy_const_generics, ["1"]]
- *neon-unstable-f16
- *target-not-arm64ec
static_defs: ['const LANE: i32']
safety: safe
types:
@ -13971,6 +14068,7 @@ intrinsics:
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, [dup]]}]]
- *neon-fp16
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- [float16x4_t, f16]
@ -14585,6 +14683,7 @@ intrinsics:
- FnCall: [cfg_attr, [*test-is-arm, { FnCall: [assert_instr, ['vbsl']]}]]
- FnCall: [cfg_attr, [*neon-target-aarch64-arm64ec, {FnCall: [assert_instr, ['bsl']]}]]
- *neon-unstable-f16
- *target-not-arm64ec
safety: safe
types:
- ['vbslq_f16', 'uint16x8_t', 'float16x8_t', 'int16x8_t::splat(-1)']

2
library/stdarch/crates/stdarch-gen-arm/src/load_store_tests.rs
View file

@ -85,7 +85,7 @@ pub fn generate_load_store_tests(
TokenStream::from_str(&PREAMBLE).map_err(|e| format!("Preamble is invalid: {e}"))?;
// Only output manual tests for the SVE set
let manual_tests = match &load_intrinsics[0].target_features[..] {
[s] if s == "sve" => TokenStream::from_str(&MANUAL_TESTS)
[s] if s == "sve" => TokenStream::from_str(MANUAL_TESTS)
.map_err(|e| format!("Manual tests are invalid: {e}"))?,
_ => quote!(),
};

6
library/stdarch/crates/stdarch-test/src/disassembly.rs
View file

@ -27,9 +27,9 @@ fn normalize(mut symbol: &str) -> String {
symbol = symbol[last_colon + 1..].to_string();
}
// Normalize to no leading underscore to handle platforms that may
// Normalize to no leading mangling chars to handle platforms that may
// inject extra ones in symbol names.
while symbol.starts_with('_') || symbol.starts_with('.') {
while symbol.starts_with('_') || symbol.starts_with('.') || symbol.starts_with('#') {
symbol.remove(0);
}
// Windows/x86 has a suffix such as @@4.
@ -49,6 +49,8 @@ pub(crate) fn disassemble_myself() -> HashSet<Function> {
"i686-pc-windows-msvc"
} else if cfg!(target_arch = "aarch64") {
"aarch64-pc-windows-msvc"
} else if cfg!(target_arch = "arm64ec") {
"arm64ec-pc-windows-msvc"
} else {
panic!("disassembly unimplemented")
};

1
library/stdarch/examples/hex.rs
View file

@ -22,7 +22,6 @@
#![allow(
clippy::unwrap_used,
clippy::print_stdout,
clippy::unwrap_used,
clippy::shadow_reuse,
clippy::cast_possible_wrap,
clippy::cast_ptr_alignment,

5
src/bootstrap/src/core/builder/cargo.rs
View file

@ -1227,6 +1227,11 @@ impl Builder<'_> {
rustflags.arg("-Zehcont-guard");
}
// Optionally override the rc.exe when compiling rustc on Windows.
if let Some(windows_rc) = &self.config.windows_rc {
cargo.env("RUSTC_WINDOWS_RC", windows_rc);
}
// For `cargo doc` invocations, make rustdoc print the Rust version into the docs
// This replaces spaces with tabs because RUSTDOCFLAGS does not
// support arguments with regular spaces. Hopefully someday Cargo will

3
src/bootstrap/src/core/config/config.rs
View file

@ -273,6 +273,7 @@ pub struct Config {
pub gdb: Option<PathBuf>,
pub lldb: Option<PathBuf>,
pub python: Option<PathBuf>,
pub windows_rc: Option<PathBuf>,
pub reuse: Option<PathBuf>,
pub cargo_native_static: bool,
pub configure_args: Vec<String>,
@ -450,6 +451,7 @@ impl Config {
nodejs: build_nodejs,
npm: build_npm,
python: build_python,
windows_rc: build_windows_rc,
reuse: build_reuse,
locked_deps: build_locked_deps,
vendor: build_vendor,
@ -1342,6 +1344,7 @@ impl Config {
.unwrap_or(rust_debug == Some(true)),
vendor,
verbose_tests,
windows_rc: build_windows_rc.map(PathBuf::from),
// tidy-alphabetical-end
}
}

1
src/bootstrap/src/core/config/toml/build.rs
View file

@ -37,6 +37,7 @@ define_config! {
nodejs: Option<String> = "nodejs",
npm: Option<String> = "npm",
python: Option<String> = "python",
windows_rc: Option<String> = "windows-rc",
reuse: Option<String> = "reuse",
locked_deps: Option<bool> = "locked-deps",
vendor: Option<bool> = "vendor",

5
src/bootstrap/src/utils/change_tracker.rs
View file

@ -551,4 +551,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[
severity: ChangeSeverity::Info,
summary: "There is now a bootstrap option called `rust.parallel-frontend-threads`, which can be used to set the number of threads for the compiler frontend used during compilation of Rust code.",
},
ChangeInfo {
change_id: 146663,
severity: ChangeSeverity::Info,
summary: "New option `build.windows-rc` that will override which resource compiler on Windows will be used to compile Rust.",
},
];

25
src/librustdoc/html/static/js/rustdoc.d.ts vendored
View file

@ -348,7 +348,7 @@ declare namespace rustdoc {
returned: rustdoc.QueryElement[],
is_alias: boolean,
alias?: string,
original?: rustdoc.Rlow,
item: rustdoc.Row,
}
/**
@ -533,4 +533,27 @@ declare namespace rustdoc {
* Generated by `render_call_locations` in `render/mod.rs`.
*/
type ScrapedLoc = [[number, number], string, string]
/**
* Each of these identifiers are used specially by
* type-driven search. Most of them are lang items
* in the compiler.
*/
type TypeNameIds = {
"typeNameIdOfOutput": number,
"typeNameIdOfFnPtr": number,
"typeNameIdOfFn": number,
"typeNameIdOfFnMut": number,
"typeNameIdOfFnOnce": number,
"typeNameIdOfArray": number,
"typeNameIdOfSlice": number,
"typeNameIdOfArrayOrSlice": number,
"typeNameIdOfTuple": number,
"typeNameIdOfUnit": number,
"typeNameIdOfTupleOrUnit": number,
"typeNameIdOfReference": number,
"typeNameIdOfPointer": number,
"typeNameIdOfHof": number,
"typeNameIdOfNever": number,
};
}

304
src/librustdoc/html/static/js/search.js
View file

@ -1190,17 +1190,6 @@ class DocSearch {
this.rootPath = rootPath;
this.database = database;
this.typeNameIdOfOutput = -1;
this.typeNameIdOfArray = -1;
this.typeNameIdOfSlice = -1;
this.typeNameIdOfArrayOrSlice = -1;
this.typeNameIdOfTuple = -1;
this.typeNameIdOfUnit = -1;
this.typeNameIdOfTupleOrUnit = -1;
this.typeNameIdOfReference = -1;
this.typeNameIdOfPointer = -1;
this.typeNameIdOfHof = -1;
this.utf8decoder = new TextDecoder();
/** @type {Map<number|null, rustdoc.FunctionType>} */
@ -1208,14 +1197,49 @@ class DocSearch {
}
/**
* Load search index. If you do not call this function, `execQuery`
* will never fulfill.
* Load type name ID set.
*
* Each of these identifiers are used specially by
* type-driven search. Most of them are lang items
* in the compiler.
*
* Use this function, which caches the result, and not
* getTypeNameIdsAsync, which is an internal implementation
* detail for this.
*
* @return {Promise<rustdoc.TypeNameIds>|rustdoc.TypeNameIds}
*/
async buildIndex() {
getTypeNameIds() {
if (this.typeNameIds) {
return this.typeNameIds;
}
const nn = this.database.getData("normalizedName");
if (!nn) {
return;
return {
typeNameIdOfOutput: -1,
typeNameIdOfFnPtr: -1,
typeNameIdOfFn: -1,
typeNameIdOfFnMut: -1,
typeNameIdOfFnOnce: -1,
typeNameIdOfArray: -1,
typeNameIdOfSlice: -1,
typeNameIdOfArrayOrSlice: -1,
typeNameIdOfTuple: -1,
typeNameIdOfUnit: -1,
typeNameIdOfTupleOrUnit: -1,
typeNameIdOfReference: -1,
typeNameIdOfPointer: -1,
typeNameIdOfHof: -1,
typeNameIdOfNever: -1,
};
}
return this.getTypeNameIdsAsync(nn);
}
/**
* @param {stringdex.DataColumn} nn
* @returns {Promise<rustdoc.TypeNameIds>}
*/
async getTypeNameIdsAsync(nn) {
// Each of these identifiers are used specially by
// type-driven search.
const [
@ -1274,21 +1298,39 @@ class DocSearch {
}
return -1;
};
this.typeNameIdOfOutput = await first(output, TY_ASSOCTYPE, "");
this.typeNameIdOfFnPtr = await first(fn, TY_PRIMITIVE, "");
this.typeNameIdOfFn = await first(fn, TY_TRAIT, "core::ops");
this.typeNameIdOfFnMut = await first(fnMut, TY_TRAIT, "core::ops");
this.typeNameIdOfFnOnce = await first(fnOnce, TY_TRAIT, "core::ops");
this.typeNameIdOfArray = await first(array, TY_PRIMITIVE, "");
this.typeNameIdOfSlice = await first(slice, TY_PRIMITIVE, "");
this.typeNameIdOfArrayOrSlice = await first(arrayOrSlice, TY_PRIMITIVE, "");
this.typeNameIdOfTuple = await first(tuple, TY_PRIMITIVE, "");
this.typeNameIdOfUnit = await first(unit, TY_PRIMITIVE, "");
this.typeNameIdOfTupleOrUnit = await first(tupleOrUnit, TY_PRIMITIVE, "");
this.typeNameIdOfReference = await first(reference, TY_PRIMITIVE, "");
this.typeNameIdOfPointer = await first(pointer, TY_PRIMITIVE, "");
this.typeNameIdOfHof = await first(hof, TY_PRIMITIVE, "");
this.typeNameIdOfNever = await first(never, TY_PRIMITIVE, "");
const typeNameIdOfOutput = await first(output, TY_ASSOCTYPE, "");
const typeNameIdOfFnPtr = await first(fn, TY_PRIMITIVE, "");
const typeNameIdOfFn = await first(fn, TY_TRAIT, "core::ops");
const typeNameIdOfFnMut = await first(fnMut, TY_TRAIT, "core::ops");
const typeNameIdOfFnOnce = await first(fnOnce, TY_TRAIT, "core::ops");
const typeNameIdOfArray = await first(array, TY_PRIMITIVE, "");
const typeNameIdOfSlice = await first(slice, TY_PRIMITIVE, "");
const typeNameIdOfArrayOrSlice = await first(arrayOrSlice, TY_PRIMITIVE, "");
const typeNameIdOfTuple = await first(tuple, TY_PRIMITIVE, "");
const typeNameIdOfUnit = await first(unit, TY_PRIMITIVE, "");
const typeNameIdOfTupleOrUnit = await first(tupleOrUnit, TY_PRIMITIVE, "");
const typeNameIdOfReference = await first(reference, TY_PRIMITIVE, "");
const typeNameIdOfPointer = await first(pointer, TY_PRIMITIVE, "");
const typeNameIdOfHof = await first(hof, TY_PRIMITIVE, "");
const typeNameIdOfNever = await first(never, TY_PRIMITIVE, "");
this.typeNameIds = {
typeNameIdOfOutput,
typeNameIdOfFnPtr,
typeNameIdOfFn,
typeNameIdOfFnMut,
typeNameIdOfFnOnce,
typeNameIdOfArray,
typeNameIdOfSlice,
typeNameIdOfArrayOrSlice,
typeNameIdOfTuple,
typeNameIdOfUnit,
typeNameIdOfTupleOrUnit,
typeNameIdOfReference,
typeNameIdOfPointer,
typeNameIdOfHof,
typeNameIdOfNever,
};
return this.typeNameIds;
}
/**
@ -1758,12 +1800,8 @@ class DocSearch {
const l = crateNames.length;
const names = [];
for (let i = 0; i < l; ++i) {
names.push(crateNames.at(i).then(name => {
if (name === undefined) {
return "";
}
return this.utf8decoder.decode(name);
}));
const name = await crateNames.at(i);
names.push(name === undefined ? "" : this.utf8decoder.decode(name));
}
return Promise.all(names);
}
@ -1820,6 +1858,9 @@ class DocSearch {
modulePathData,
exactModuleName,
exactModulePathData,
parentName,
parentPath,
crate,
] = await Promise.all([
entry && entry.modulePath !== null ? this.getName(entry.modulePath) : null,
entry && entry.modulePath !== null ? this.getPathData(entry.modulePath) : null,
@ -1829,6 +1870,13 @@ class DocSearch {
entry && entry.exactModulePath !== null ?
this.getPathData(entry.exactModulePath) :
null,
entry && entry.parent !== null ?
this.getName(entry.parent) :
null,
entry && entry.parent !== null ?
this.getPathData(entry.parent) :
null,
entry ? nonnull(await this.getName(entry.krate)) : "",
]);
const name = name_ === null ? "" : name_;
const normalizedName = (name.indexOf("_") === -1 ?
@ -1838,12 +1886,9 @@ class DocSearch {
(modulePathData.modulePath === "" ?
moduleName :
`${modulePathData.modulePath}::${moduleName}`);
const [parentName, parentPath] = entry !== null && entry.parent !== null ?
await Promise.all([this.getName(entry.parent), this.getPathData(entry.parent)]) :
[null, null];
return {
id,
crate: entry ? nonnull(await this.getName(entry.krate)) : "",
crate,
ty: entry ? entry.ty : nonnull(path).ty,
name,
normalizedName,
@ -2148,6 +2193,7 @@ class DocSearch {
* @returns {Promise<rustdoc.DisplayTypeSignature>}
*/
const formatDisplayTypeSignature = async(obj, typeInfo, elems, returned) => {
const typeNameIds = await this.getTypeNameIds();
const objType = obj.type;
if (!objType) {
return {type: [], mappedNames: new Map(), whereClause: new Map()};
@ -2173,10 +2219,12 @@ class DocSearch {
return true;
},
0,
typeNameIds,
);
return !!fnOutput;
},
0,
typeNameIds,
);
} else {
const highlighted = unifyFunctionTypes(
@ -2189,6 +2237,7 @@ class DocSearch {
return true;
},
0,
typeNameIds,
);
if (typeInfo === "elems") {
fnInputs = highlighted;
@ -2268,7 +2317,7 @@ class DocSearch {
* @returns {Promise<void>}
*/
const writeHof = async(fnType, result) => {
const hofOutput = fnType.bindings.get(this.typeNameIdOfOutput) || [];
const hofOutput = fnType.bindings.get(typeNameIds.typeNameIdOfOutput) || [];
const hofInputs = fnType.generics;
pushText(fnType, result);
pushText({name: " (", highlighted: false}, result);
@ -2309,11 +2358,14 @@ class DocSearch {
* @returns {Promise<boolean>}
*/
const writeSpecialPrimitive = async(fnType, result) => {
if (fnType.id === this.typeNameIdOfArray || fnType.id === this.typeNameIdOfSlice ||
fnType.id === this.typeNameIdOfTuple || fnType.id === this.typeNameIdOfUnit) {
if (fnType.id === typeNameIds.typeNameIdOfArray ||
fnType.id === typeNameIds.typeNameIdOfSlice ||
fnType.id === typeNameIds.typeNameIdOfTuple ||
fnType.id === typeNameIds.typeNameIdOfUnit
) {
const [ob, sb] =
fnType.id === this.typeNameIdOfArray ||
fnType.id === this.typeNameIdOfSlice ?
fnType.id === typeNameIds.typeNameIdOfArray ||
fnType.id === typeNameIds.typeNameIdOfSlice ?
["[", "]"] :
["(", ")"];
pushText({ name: ob, highlighted: fnType.highlighted }, result);
@ -2325,7 +2377,7 @@ class DocSearch {
);
pushText({ name: sb, highlighted: fnType.highlighted }, result);
return true;
} else if (fnType.id === this.typeNameIdOfReference) {
} else if (fnType.id === typeNameIds.typeNameIdOfReference) {
pushText({ name: "&", highlighted: fnType.highlighted }, result);
let prevHighlighted = false;
await onEachBtwnAsync(
@ -2341,7 +2393,7 @@ class DocSearch {
}, result),
);
return true;
} else if (fnType.id === this.typeNameIdOfPointer) {
} else if (fnType.id === typeNameIds.typeNameIdOfPointer) {
pushText({ name: "*", highlighted: fnType.highlighted }, result);
if (fnType.generics.length < 2) {
pushText({ name: "const ", highlighted: fnType.highlighted }, result);
@ -2361,14 +2413,14 @@ class DocSearch {
);
return true;
} else if (
fnType.id === this.typeNameIdOfFn ||
fnType.id === this.typeNameIdOfFnMut ||
fnType.id === this.typeNameIdOfFnOnce ||
fnType.id === this.typeNameIdOfFnPtr
fnType.id === typeNameIds.typeNameIdOfFn ||
fnType.id === typeNameIds.typeNameIdOfFnMut ||
fnType.id === typeNameIds.typeNameIdOfFnOnce ||
fnType.id === typeNameIds.typeNameIdOfFnPtr
) {
await writeHof(fnType, result);
return true;
} else if (fnType.id === this.typeNameIdOfNever) {
} else if (fnType.id === typeNameIds.typeNameIdOfNever) {
pushText({ name: "!", highlighted: fnType.highlighted }, result);
return true;
}
@ -2426,10 +2478,10 @@ class DocSearch {
return;
}
} else if (fnType.ty === TY_TRAIT && (
fnType.id === this.typeNameIdOfFn ||
fnType.id === this.typeNameIdOfFnMut ||
fnType.id === this.typeNameIdOfFnOnce ||
fnType.id === this.typeNameIdOfFnPtr
fnType.id === typeNameIds.typeNameIdOfFn ||
fnType.id === typeNameIds.typeNameIdOfFnMut ||
fnType.id === typeNameIds.typeNameIdOfFnOnce ||
fnType.id === typeNameIds.typeNameIdOfFnPtr
)) {
await writeHof(fnType, result);
return;
@ -2540,7 +2592,7 @@ class DocSearch {
* Add extra data to result objects, and filter items that have been
* marked for removal.
*
* @param {[rustdoc.PlainResultObject, rustdoc.Row][]} results
* @param {rustdoc.PlainResultObject[]} results
* @param {"sig"|"elems"|"returned"|null} typeInfo
* @param {Set<string>} duplicates
* @returns {rustdoc.ResultObject[]}
@ -2548,7 +2600,8 @@ class DocSearch {
const transformResults = (results, typeInfo, duplicates) => {
const out = [];
for (const [result, item] of results) {
for (const result of results) {
const item = result.item;
if (item.id !== -1) {
const res = buildHrefAndPath(item);
// many of these properties don't strictly need to be
@ -2633,7 +2686,7 @@ class DocSearch {
const normalizedUserQuery = parsedQuery.userQuery.toLowerCase();
const isMixedCase = normalizedUserQuery !== userQuery;
/**
* @type {[rustdoc.PlainResultObject, rustdoc.Row][]}
* @type {rustdoc.PlainResultObject[]}
*/
const result_list = [];
for (const result of results.values()) {
@ -2641,23 +2694,22 @@ class DocSearch {
continue;
}
/**
* @type {rustdoc.Row?}
* @type {rustdoc.Row}
*/
const item = await this.getRow(result.id, typeInfo !== null);
if (!item) {
continue;
}
const item = result.item;
if (filterCrates !== null && item.crate !== filterCrates) {
continue;
}
if (item) {
result_list.push([result, item]);
result_list.push(result);
} else {
continue;
}
}
result_list.sort(([aaa, aai], [bbb, bbi]) => {
result_list.sort((aaa, bbb) => {
const aai = aaa.item;
const bbi = bbb.item;
/** @type {number} */
let a;
/** @type {number} */
@ -2804,6 +2856,7 @@ class DocSearch {
* @param {number} unboxingDepth
* - Limit checks that Ty matches Vec<Ty>,
* but not Vec<ParamEnvAnd<WithInfcx<ConstTy<Interner<Ty=Ty>>>>>
* @param {rustdoc.TypeNameIds} typeNameIds
*
* @return {rustdoc.HighlightedFunctionType[]|null}
* - Returns highlighted results if a match, null otherwise.
@ -2815,6 +2868,7 @@ class DocSearch {
mgensIn,
solutionCb,
unboxingDepth,
typeNameIds,
) {
if (unboxingDepth >= UNBOXING_LIMIT) {
return null;
@ -2837,7 +2891,12 @@ class DocSearch {
&& queryElems[0].bindings.size === 0) {
const queryElem = queryElems[0];
for (const [i, fnType] of fnTypesIn.entries()) {
if (!unifyFunctionTypeIsMatchCandidate(fnType, queryElem, mgens)) {
if (!unifyFunctionTypeIsMatchCandidate(
fnType,
queryElem,
mgens,
typeNameIds,
)) {
continue;
}
if (fnType.id !== null &&
@ -2873,6 +2932,7 @@ class DocSearch {
mgens ? new Map(mgens) : null,
solutionCb,
unboxingDepth,
typeNameIds,
) || fnType.generics,
});
return highlighted;
@ -2885,6 +2945,7 @@ class DocSearch {
whereClause,
mgens,
unboxingDepth + 1,
typeNameIds,
)) {
continue;
}
@ -2898,6 +2959,7 @@ class DocSearch {
mgens,
solutionCb,
unboxingDepth + 1,
typeNameIds,
);
if (highlightedGenerics) {
const highlighted = [...fnTypesIn];
@ -2916,6 +2978,7 @@ class DocSearch {
mgens ? new Map(mgens) : null,
solutionCb,
unboxingDepth + 1,
typeNameIds,
);
if (highlightedGenerics) {
const highlighted = [...fnTypesIn];
@ -2960,7 +3023,12 @@ class DocSearch {
let queryElemsTmp = null;
for (let i = flast; i >= 0; i -= 1) {
const fnType = fnTypes[i];
if (!unifyFunctionTypeIsMatchCandidate(fnType, queryElem, mgens)) {
if (!unifyFunctionTypeIsMatchCandidate(
fnType,
queryElem,
mgens,
typeNameIds,
)) {
continue;
}
let mgensScratch;
@ -3004,6 +3072,7 @@ class DocSearch {
whereClause,
mgensScratch,
unboxingDepth,
typeNameIds,
);
if (!solution) {
return false;
@ -3017,6 +3086,7 @@ class DocSearch {
simplifiedMgens,
solutionCb,
unboxingDepth,
typeNameIds,
);
if (unifiedGenerics !== null) {
unifiedGenericsMgens = simplifiedMgens;
@ -3026,6 +3096,7 @@ class DocSearch {
return false;
},
unboxingDepth,
typeNameIds,
);
if (passesUnification) {
passesUnification.length = fl;
@ -3042,6 +3113,7 @@ class DocSearch {
unifiedGenericsMgens,
solutionCb,
unboxingDepth,
typeNameIds,
// @ts-expect-error
) : unifiedGenerics.splice(0, v.length)];
})),
@ -3061,6 +3133,7 @@ class DocSearch {
whereClause,
mgens,
unboxingDepth + 1,
typeNameIds,
)) {
continue;
}
@ -3077,6 +3150,7 @@ class DocSearch {
mgens,
solutionCb,
unboxingDepth + 1,
typeNameIds,
);
if (passesUnification) {
const highlightedGenerics = passesUnification.slice(
@ -3117,6 +3191,7 @@ class DocSearch {
* @param {number} unboxingDepth
* - Limit checks that Ty matches Vec<Ty>,
* but not Vec<ParamEnvAnd<WithInfcx<ConstTy<Interner<Ty=Ty>>>>>
* @param {rustdoc.TypeNameIds} typeNameIds
*
* @return {rustdoc.HighlightedFunctionType[]|null}
* - Returns highlighted results if a match, null otherwise.
@ -3128,6 +3203,7 @@ class DocSearch {
mgensIn,
solutionCb,
unboxingDepth,
typeNameIds,
) {
if (unboxingDepth >= UNBOXING_LIMIT) {
return null;
@ -3145,7 +3221,12 @@ class DocSearch {
}
const fnType = fnTypesIn[0];
const queryElem = queryElems[0];
if (unifyFunctionTypeIsMatchCandidate(fnType, queryElem, mgens)) {
if (unifyFunctionTypeIsMatchCandidate(
fnType,
queryElem,
mgens,
typeNameIds,
)) {
if (fnType.id !== null &&
fnType.id < 0 &&
queryElem.id !== null &&
@ -3163,6 +3244,7 @@ class DocSearch {
mgensScratch,
solutionCb,
unboxingDepth,
typeNameIds,
);
if (fnTypesRemaining) {
const highlighted = [fnType, ...fnTypesRemaining];
@ -3189,6 +3271,7 @@ class DocSearch {
whereClause,
mgensScratch,
unboxingDepth,
typeNameIds,
);
if (!solution) {
return false;
@ -3202,6 +3285,7 @@ class DocSearch {
simplifiedMgens,
solutionCb,
unboxingDepth,
typeNameIds,
);
if (unifiedGenerics !== null) {
return true;
@ -3209,6 +3293,7 @@ class DocSearch {
}
},
unboxingDepth,
typeNameIds,
);
if (fnTypesRemaining) {
const highlighted = [fnType, ...fnTypesRemaining];
@ -3227,6 +3312,7 @@ class DocSearch {
whereClause,
mgens,
unboxingDepth + 1,
typeNameIds,
)) {
let highlightedRemaining;
if (fnType.id !== null && fnType.id < 0) {
@ -3248,6 +3334,7 @@ class DocSearch {
mgensScratch,
solutionCb,
unboxingDepth,
typeNameIds,
);
if (hl) {
highlightedRemaining = hl;
@ -3255,6 +3342,7 @@ class DocSearch {
return hl;
},
unboxingDepth + 1,
typeNameIds,
);
if (highlightedGenerics) {
return [Object.assign({
@ -3282,6 +3370,7 @@ class DocSearch {
mgensScratch,
solutionCb,
unboxingDepth,
typeNameIds,
);
if (hl) {
highlightedRemaining = hl;
@ -3289,6 +3378,7 @@ class DocSearch {
return hl;
},
unboxingDepth + 1,
typeNameIds,
);
if (highlightedGenerics) {
return [Object.assign({}, fnType, {
@ -3314,10 +3404,11 @@ class DocSearch {
*
* @param {rustdoc.FunctionType} fnType
* @param {rustdoc.QueryElement} queryElem
* @param {rustdoc.TypeNameIds} typeNameIds
* @param {Map<number,number>|null} mgensIn - Map query generics to function generics.
* @returns {boolean}
*/
const unifyFunctionTypeIsMatchCandidate = (fnType, queryElem, mgensIn) => {
const unifyFunctionTypeIsMatchCandidate = (fnType, queryElem, mgensIn, typeNameIds) => {
// type filters look like `trait:Read` or `enum:Result`
if (!typePassesFilter(queryElem.typeFilter, fnType.ty)) {
return false;
@ -3336,21 +3427,23 @@ class DocSearch {
} else {
// For these special cases, matching code need added to the inverted index.
// search_index.rs -> convert_render_type does this
if (queryElem.id === this.typeNameIdOfArrayOrSlice &&
(fnType.id === this.typeNameIdOfSlice || fnType.id === this.typeNameIdOfArray)
if (queryElem.id === typeNameIds.typeNameIdOfArrayOrSlice &&
(fnType.id === typeNameIds.typeNameIdOfSlice ||
fnType.id === typeNameIds.typeNameIdOfArray)
) {
// [] matches primitive:array or primitive:slice
// if it matches, then we're fine, and this is an appropriate match candidate
} else if (queryElem.id === this.typeNameIdOfTupleOrUnit &&
(fnType.id === this.typeNameIdOfTuple || fnType.id === this.typeNameIdOfUnit)
} else if (queryElem.id === typeNameIds.typeNameIdOfTupleOrUnit &&
(fnType.id === typeNameIds.typeNameIdOfTuple ||
fnType.id === typeNameIds.typeNameIdOfUnit)
) {
// () matches primitive:tuple or primitive:unit
// if it matches, then we're fine, and this is an appropriate match candidate
} else if (queryElem.id === this.typeNameIdOfHof && (
fnType.id === this.typeNameIdOfFn ||
fnType.id === this.typeNameIdOfFnMut ||
fnType.id === this.typeNameIdOfFnOnce ||
fnType.id === this.typeNameIdOfFnPtr
} else if (queryElem.id === typeNameIds.typeNameIdOfHof && (
fnType.id === typeNameIds.typeNameIdOfFn ||
fnType.id === typeNameIds.typeNameIdOfFnMut ||
fnType.id === typeNameIds.typeNameIdOfFnOnce ||
fnType.id === typeNameIds.typeNameIdOfFnPtr
)) {
// -> matches fn, fnonce, and fnmut
// if it matches, then we're fine, and this is an appropriate match candidate
@ -3414,6 +3507,7 @@ class DocSearch {
* @param {Map<number,number>|null} mgensIn - Map query generics to function generics.
* Never modified.
* @param {number} unboxingDepth
* @param {rustdoc.TypeNameIds} typeNameIds
* @returns {false|{
* mgens: [Map<number,number>|null], simplifiedGenerics: rustdoc.FunctionType[]
* }}
@ -3424,6 +3518,7 @@ class DocSearch {
whereClause,
mgensIn,
unboxingDepth,
typeNameIds,
) {
if (fnType.bindings.size < queryElem.bindings.size) {
return false;
@ -3455,6 +3550,7 @@ class DocSearch {
return false;
},
unboxingDepth,
typeNameIds,
);
return newSolutions;
});
@ -3486,6 +3582,7 @@ class DocSearch {
* @param {rustdoc.FunctionType[][]} whereClause - Trait bounds for generic items.
* @param {Map<number,number>|null} mgens - Map query generics to function generics.
* @param {number} unboxingDepth
* @param {rustdoc.TypeNameIds} typeNameIds
* @returns {boolean}
*/
function unifyFunctionTypeIsUnboxCandidate(
@ -3494,6 +3591,7 @@ class DocSearch {
whereClause,
mgens,
unboxingDepth,
typeNameIds,
) {
if (unboxingDepth >= UNBOXING_LIMIT) {
return false;
@ -3512,6 +3610,7 @@ class DocSearch {
whereClause,
mgens,
unboxingDepth,
typeNameIds,
);
} else if (fnType.unboxFlag &&
(fnType.generics.length > 0 || fnType.bindings.size > 0)) {
@ -3525,6 +3624,7 @@ class DocSearch {
whereClause,
mgens,
unboxingDepth,
typeNameIds,
);
}
return false;
@ -3537,14 +3637,15 @@ class DocSearch {
* @param {rustdoc.QueryElement[]} elems
* @param {rustdoc.FunctionType[]} list - A list of function types.
* @param {rustdoc.FunctionType[][]} where_clause - Trait bounds for generic items.
* @param {rustdoc.TypeNameIds} typeNameIds
*/
function containsTypeFromQuery(elems, list, where_clause) {
function containsTypeFromQuery(elems, list, where_clause, typeNameIds) {
if (!list) return false;
for (const ty of elems) {
if (ty.id !== null && ty.id < 0) {
continue;
}
if (checkIfInList(list, ty, where_clause, null, 0)) {
if (checkIfInList(list, ty, where_clause, null, 0, typeNameIds)) {
return true;
}
}
@ -3560,12 +3661,13 @@ class DocSearch {
* @param {rustdoc.FunctionType[][]} whereClause - Trait bounds for generic items.
* @param {Map<number,number>|null} mgens - Map functions generics to query generics.
* @param {number} unboxingDepth
* @param {rustdoc.TypeNameIds} typeNameIds
*
* @return {boolean} - Returns true if found, false otherwise.
*/
function checkIfInList(list, elem, whereClause, mgens, unboxingDepth) {
function checkIfInList(list, elem, whereClause, mgens, unboxingDepth, typeNameIds) {
for (const entry of list) {
if (checkType(entry, elem, whereClause, mgens, unboxingDepth)) {
if (checkType(entry, elem, whereClause, mgens, unboxingDepth, typeNameIds)) {
return true;
}
}
@ -3580,11 +3682,12 @@ class DocSearch {
* @param {rustdoc.QueryElement} elem - The element from the parsed query.
* @param {rustdoc.FunctionType[][]} whereClause - Trait bounds for generic items.
* @param {Map<number,number>|null} mgens - Map query generics to function generics.
* @param {number} unboxingDepth
* @param {rustdoc.TypeNameIds} typeNameIds
*
* @return {boolean} - Returns true if the type matches, false otherwise.
*/
// @ts-expect-error
const checkType = (row, elem, whereClause, mgens, unboxingDepth) => {
const checkType = (row, elem, whereClause, mgens, unboxingDepth, typeNameIds) => {
if (unboxingDepth >= UNBOXING_LIMIT) {
return false;
}
@ -3593,9 +3696,9 @@ class DocSearch {
row.generics.length === 0 && elem.generics.length === 0 &&
row.bindings.size === 0 && elem.bindings.size === 0 &&
// special case
elem.id !== this.typeNameIdOfArrayOrSlice &&
elem.id !== this.typeNameIdOfHof &&
elem.id !== this.typeNameIdOfTupleOrUnit
elem.id !== typeNameIds.typeNameIdOfArrayOrSlice &&
elem.id !== typeNameIds.typeNameIdOfHof &&
elem.id !== typeNameIds.typeNameIdOfTupleOrUnit
) {
return row.id === elem.id && typePassesFilter(elem.typeFilter, row.ty);
} else {
@ -3607,6 +3710,7 @@ class DocSearch {
mgens,
() => true,
unboxingDepth,
typeNameIds,
);
}
};
@ -3673,7 +3777,10 @@ class DocSearch {
/**
* Compute an "edit distance" that ignores missing path elements.
* @param {string[]} contains search query path
* @param {rustdoc.Row} row indexed item
* @param {{
* modulePath: string,
* parent: { path: rustdoc.PathData, name: string}?,
* }} row indexed item
* @returns {null|number} edit distance
*/
function checkRowPath(contains, row) {
@ -3752,7 +3859,7 @@ class DocSearch {
is_alias: true,
elems: [], // only used in type-based queries
returned: [], // only used in type-based queries
original: await this.getRow(alias, false),
item: nonnull(await this.getRow(alias, false)),
};
};
/**
@ -3834,6 +3941,7 @@ class DocSearch {
elems: [], // only used in type-based queries
returned: [], // only used in type-based queries
is_alias: false,
item: row,
} : null;
} else {
return {
@ -3848,6 +3956,7 @@ class DocSearch {
elems: [], // only used in type-based queries
returned: [], // only used in type-based queries
is_alias: false,
item: row,
};
}
};
@ -4359,9 +4468,10 @@ class DocSearch {
};
// finally, we can do the actual unification loop
const [allInputs, allOutput] = await Promise.all([
const [allInputs, allOutput, typeNameIds] = await Promise.all([
unpackPostingsListAll(inputs, "invertedFunctionInputsIndex"),
unpackPostingsListAll(output, "invertedFunctionOutputIndex"),
this.getTypeNameIds(),
]);
let checkCounter = 0;
/**
@ -4430,12 +4540,18 @@ class DocSearch {
mgens,
checkTypeMgensForConflict,
0, // unboxing depth
typeNameIds,
);
},
0, // unboxing depth
typeNameIds,
)) {
return null;
}
const item = await this.getRow(id, true);
if (!item) {
return null;
}
const result = {
id,
dist: fnData.elemCount,
@ -4444,8 +4560,9 @@ class DocSearch {
elems: inputs,
returned: output,
is_alias: false,
item,
};
const entry = await this.getEntryData(id);
const entry = item.entry;
if ((entry && !isFnLikeTy(entry.ty)) ||
(isReturnTypeQuery &&
functionSignature &&
@ -4453,6 +4570,7 @@ class DocSearch {
output,
functionSignature.inputs,
functionSignature.where_clause,
typeNameIds,
)
)
) {
@ -5273,7 +5391,6 @@ if (ROOT_PATH === null) {
const database = await Stringdex.loadDatabase(hooks);
if (typeof window !== "undefined") {
docSearch = new DocSearch(ROOT_PATH, database);
await docSearch.buildIndex();
onEachLazy(document.querySelectorAll(
".search-form.loading",
), form => {
@ -5286,7 +5403,6 @@ if (typeof window !== "undefined") {
}
} else if (typeof exports !== "undefined") {
docSearch = new DocSearch(ROOT_PATH, database);
await docSearch.buildIndex();
return { docSearch, DocSearch };
}
};

2
src/librustdoc/html/static/js/stringdex.d.ts vendored
View file

@ -29,7 +29,7 @@ declare namespace stringdex {
*/
interface DataColumn {
isEmpty(id: number): boolean;
at(id: number): Promise<Uint8Array|undefined>;
at(id: number): Promise<Uint8Array>|Uint8Array|undefined;
search(name: Uint8Array|string): Promise<Trie?>;
searchLev(name: Uint8Array|string): AsyncGenerator<Trie>;
length: number,

101
src/librustdoc/html/static/js/stringdex.js
View file

@ -2261,7 +2261,7 @@ function loadDatabase(hooks) {
this.hashes = hashes;
this.emptyset = emptyset;
this.name = name;
/** @type {{"hash": Uint8Array, "data": Promise<Uint8Array[]>?, "end": number}[]} */
/** @type {{"hash": Uint8Array, "data": Uint8Array[]?, "end": number}[]} */
this.buckets = [];
this.bucket_keys = [];
const l = counts.length;
@ -2295,65 +2295,76 @@ function loadDatabase(hooks) {
/**
* Look up a cell by row ID.
* @param {number} id
* @returns {Promise<Uint8Array|undefined>}
* @returns {Promise<Uint8Array>|Uint8Array|undefined}
*/
async at(id) {
at(id) {
if (this.emptyset.contains(id)) {
return Promise.resolve(EMPTY_UINT8);
return EMPTY_UINT8;
} else {
let idx = -1;
while (this.bucket_keys[idx + 1] <= id) {
idx += 1;
}
if (idx === -1 || idx >= this.bucket_keys.length) {
return Promise.resolve(undefined);
return undefined;
} else {
const start = this.bucket_keys[idx];
const {hash, end} = this.buckets[idx];
let data = this.buckets[idx].data;
const bucket = this.buckets[idx];
const data = this.buckets[idx].data;
if (data === null) {
const dataSansEmptysetOrig = await registry.dataLoadByNameAndHash(
this.name,
hash,
);
// After the `await` resolves, another task might fill
// in the data. If so, we should use that.
data = this.buckets[idx].data;
if (data !== null) {
return (await data)[id - start];
}
const dataSansEmptyset = [...dataSansEmptysetOrig];
/** @type {(Uint8Array[])|null} */
let dataWithEmptyset = null;
let pos = start;
let insertCount = 0;
while (pos < end) {
if (this.emptyset.contains(pos)) {
if (dataWithEmptyset === null) {
dataWithEmptyset = dataSansEmptyset.splice(0, insertCount);
} else if (insertCount !== 0) {
dataWithEmptyset.push(
...dataSansEmptyset.splice(0, insertCount),
);
}
insertCount = 0;
dataWithEmptyset.push(EMPTY_UINT8);
} else {
insertCount += 1;
}
pos += 1;
}
data = Promise.resolve(
dataWithEmptyset === null ?
dataSansEmptyset :
dataWithEmptyset.concat(dataSansEmptyset),
);
this.buckets[idx].data = data;
return this.atAsyncFetch(id, start, bucket);
} else {
return data[id - start];
}
return (await data)[id - start];
}
}
}
/**
* Look up a cell by row ID.
* @param {number} id
* @param {number} start
* @param {{hash: Uint8Array, data: Uint8Array[] | null, end: number}} bucket
* @returns {Promise<Uint8Array>}
*/
async atAsyncFetch(id, start, bucket) {
const {hash, end} = bucket;
const dataSansEmptysetOrig = await registry.dataLoadByNameAndHash(
this.name,
hash,
);
// After the `await` resolves, another task might fill
// in the data. If so, we should use that.
let data = bucket.data;
if (data !== null) {
return data[id - start];
}
const dataSansEmptyset = [...dataSansEmptysetOrig];
/** @type {(Uint8Array[])|null} */
let dataWithEmptyset = null;
let pos = start;
let insertCount = 0;
while (pos < end) {
if (this.emptyset.contains(pos)) {
if (dataWithEmptyset === null) {
dataWithEmptyset = dataSansEmptyset.splice(0, insertCount);
} else if (insertCount !== 0) {
dataWithEmptyset.push(
...dataSansEmptyset.splice(0, insertCount),
);
}
insertCount = 0;
dataWithEmptyset.push(EMPTY_UINT8);
} else {
insertCount += 1;
}
pos += 1;
}
data = dataWithEmptyset === null ?
dataSansEmptyset :
dataWithEmptyset.concat(dataSansEmptyset);
bucket.data = data;
return data[id - start];
}
/**
* Search by exact substring
* @param {Uint8Array|string} name

33
src/tools/miri/.github/workflows/ci.yml vendored
View file

@ -31,21 +31,22 @@ jobs:
os: ubuntu-24.04-arm
multiarch: armhf
gcc_cross: arm-linux-gnueabihf
- host_target: riscv64gc-unknown-linux-gnu
os: ubuntu-latest
multiarch: riscv64
gcc_cross: riscv64-linux-gnu
qemu: true
- host_target: s390x-unknown-linux-gnu
os: ubuntu-latest
multiarch: s390x
gcc_cross: s390x-linux-gnu
qemu: true
- host_target: powerpc64le-unknown-linux-gnu
os: ubuntu-latest
multiarch: ppc64el
gcc_cross: powerpc64le-linux-gnu
qemu: true
# Disabled due to Ubuntu repo trouble
# - host_target: riscv64gc-unknown-linux-gnu
# os: ubuntu-latest
# multiarch: riscv64
# gcc_cross: riscv64-linux-gnu
# qemu: true
# - host_target: s390x-unknown-linux-gnu
# os: ubuntu-latest
# multiarch: s390x
# gcc_cross: s390x-linux-gnu
# qemu: true
# - host_target: powerpc64le-unknown-linux-gnu
# os: ubuntu-latest
# multiarch: ppc64el
# gcc_cross: powerpc64le-linux-gnu
# qemu: true
- host_target: aarch64-apple-darwin
os: macos-latest
- host_target: i686-pc-windows-msvc
@ -67,7 +68,7 @@ jobs:
- name: install multiarch
if: ${{ matrix.multiarch != '' }}
run: |
# s390x, ppc64el need Ubuntu Ports to be in the mirror list
# s390x, ppc64el, riscv64 need Ubuntu Ports to be in the mirror list
sudo bash -c "echo 'https://ports.ubuntu.com/ priority:4' >> /etc/apt/apt-mirrors.txt"
# Add architecture
sudo dpkg --add-architecture ${{ matrix.multiarch }}

24
src/tools/miri/Cargo.lock
View file

@ -363,9 +363,9 @@ dependencies = [
[[package]]
name = "cxx"
version = "1.0.161"
version = "1.0.173"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3523cc02ad831111491dd64b27ad999f1ae189986728e477604e61b81f828df"
checksum = "6c64ed3da1c337cbaae7223cdcff8b4dddf698d188cd3eaddd1116f6b0295950"
dependencies = [
"cc",
"cxxbridge-cmd",
@ -377,9 +377,9 @@ dependencies = [
[[package]]
name = "cxx-build"
version = "1.0.161"
version = "1.0.173"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "212b754247a6f07b10fa626628c157593f0abf640a3dd04cce2760eca970f909"
checksum = "ae0a26a75a05551f5ae3d75b3557543d06682284eaa7419113162d602cb45766"
dependencies = [
"cc",
"codespan-reporting",
@ -392,9 +392,9 @@ dependencies = [
[[package]]
name = "cxxbridge-cmd"
version = "1.0.161"
version = "1.0.173"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f426a20413ec2e742520ba6837c9324b55ffac24ead47491a6e29f933c5b135a"
checksum = "952d408b6002b7db4b36da07c682a9cbb34f2db0efa03e976ae50a388414e16c"
dependencies = [
"clap",
"codespan-reporting",
@ -406,15 +406,15 @@ dependencies = [
[[package]]
name = "cxxbridge-flags"
version = "1.0.161"
version = "1.0.173"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a258b6069020b4e5da6415df94a50ee4f586a6c38b037a180e940a43d06a070d"
checksum = "ccbd201b471c75c6abb6321cace706d1982d270e308b891c11a3262d320f5265"
[[package]]
name = "cxxbridge-macro"
version = "1.0.161"
version = "1.0.173"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8dec184b52be5008d6eaf7e62fc1802caf1ad1227d11b3b7df2c409c7ffc3f4"
checksum = "2bea8b915bbc4cb4288f242aa7ca18b23ecc6965e4d6e7c1b07905e3fe2e0c41"
dependencies = [
"indexmap",
"proc-macro2",
@ -501,9 +501,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foldhash"
version = "0.1.5"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "form_urlencoded"

2
src/tools/miri/Cargo.toml
View file

@ -70,7 +70,7 @@ harness = false
[features]
default = ["stack-cache", "native-lib"]
genmc = ["dep:genmc-sys"] # this enables a GPL dependency!
genmc = ["dep:genmc-sys"]
stack-cache = []
stack-cache-consistency-check = ["stack-cache"]
tracing = ["serde_json"]

2
src/tools/miri/README.md
View file

@ -220,7 +220,7 @@ degree documented below):
- `solaris` / `illumos`: maintained by @devnexen. Supports the entire test suite.
- `freebsd`: maintained by @YohDeadfall and @LorrensP-2158466. Supports the entire test suite.
- `android`: **maintainer wanted**. Support very incomplete, but a basic "hello world" works.
- `wasi`: **maintainer wanted**. Support very incomplete, not even standard output works, but an empty `main` function works.
- `wasi`: **maintainer wanted**. Support very incomplete, but a basic "hello world" works.
- For targets on other operating systems, Miri might fail before even reaching the `main` function.
However, even for targets that we do support, the degree of support for accessing platform APIs

21
src/tools/miri/ci/ci.sh
View file

@ -137,6 +137,7 @@ function run_tests_minimal {
# In particular, fully cover all tier 1 targets.
# We also want to run the many-seeds tests on all tier 1 targets.
# We run GC_STRESS only once for each tier 1 OS.
case $HOST_TARGET in
x86_64-unknown-linux-gnu)
# Host
@ -147,29 +148,31 @@ case $HOST_TARGET in
;;
i686-unknown-linux-gnu)
# Host
# Without GC_STRESS as this is a slow runner.
MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
# Partially supported targets (tier 2)
BASIC="empty_main integer heap_alloc libc-mem vec string btreemap" # ensures we have the basics: pre-main code, system allocator
UNIX="hello panic/panic panic/unwind concurrency/simple atomic libc-mem libc-misc libc-random env num_cpus" # the things that are very similar across all Unixes, and hence easily supported there
TEST_TARGET=aarch64-linux-android run_tests_minimal $BASIC $UNIX time hashmap random thread sync concurrency epoll eventfd
TEST_TARGET=wasm32-wasip2 run_tests_minimal $BASIC wasm
TEST_TARGET=wasm32-wasip2 run_tests_minimal $BASIC hello wasm
TEST_TARGET=wasm32-unknown-unknown run_tests_minimal no_std empty_main wasm # this target doesn't really have std
TEST_TARGET=thumbv7em-none-eabihf run_tests_minimal no_std
;;
aarch64-unknown-linux-gnu)
# Host
GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
# Extra tier 2
MANY_SEEDS=16 TEST_TARGET=arm-unknown-linux-gnueabi run_tests # 32bit ARM
MANY_SEEDS=16 TEST_TARGET=aarch64-pc-windows-gnullvm run_tests # gnullvm ABI
MANY_SEEDS=16 TEST_TARGET=s390x-unknown-linux-gnu run_tests # big-endian architecture of choice
# Custom target JSON file
TEST_TARGET=tests/x86_64-unknown-kernel.json MIRI_NO_STD=1 run_tests_minimal no_std
# Not officially supported tier 2
MANY_SEEDS=16 TEST_TARGET=x86_64-unknown-freebsd run_tests
MANY_SEEDS=16 TEST_TARGET=i686-unknown-freebsd run_tests
;;
armv7-unknown-linux-gnueabihf)
# Host
GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
# Custom target JSON file
TEST_TARGET=tests/x86_64-unknown-kernel.json MIRI_NO_STD=1 run_tests_minimal no_std
;;
aarch64-apple-darwin)
# Host
@ -181,12 +184,9 @@ case $HOST_TARGET in
MANY_SEEDS=16 TEST_TARGET=mips-unknown-linux-gnu run_tests # a 32bit big-endian target, and also a target without 64bit atomics
MANY_SEEDS=16 TEST_TARGET=x86_64-unknown-illumos run_tests
MANY_SEEDS=16 TEST_TARGET=x86_64-pc-solaris run_tests
MANY_SEEDS=16 TEST_TARGET=x86_64-unknown-freebsd run_tests
MANY_SEEDS=16 TEST_TARGET=i686-unknown-freebsd run_tests
;;
i686-pc-windows-msvc)
# Host
# Without GC_STRESS as this is a very slow runner.
MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 run_tests
# Extra tier 1
# We really want to ensure a Linux target works on a Windows host,
@ -195,8 +195,7 @@ case $HOST_TARGET in
;;
aarch64-pc-windows-msvc)
# Host
# Without GC_STRESS as this is a very slow runner.
MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
# Extra tier 1
MANY_SEEDS=64 TEST_TARGET=i686-unknown-linux-gnu run_tests
;;

38
src/tools/miri/doc/genmc.md
View file

@ -9,9 +9,7 @@ Miri-GenMC integrates that model checker into Miri.
## Usage
**IMPORTANT: The license of GenMC and thus the `genmc-sys` crate in the Miri repo is currently "GPL-3.0-or-later", so a binary produced with the `genmc` feature is subject to the requirements of the GPL. As long as that remains the case, the `genmc` feature of Miri is OFF-BY-DEFAULT and must be OFF for all Miri releases.**
For testing/developing Miri-GenMC (while keeping in mind the licensing issues):
For testing/developing Miri-GenMC:
- clone the Miri repo.
- build Miri-GenMC with `./miri build --features=genmc`.
- OR: install Miri-GenMC in the current system with `./miri install --features=genmc`
@ -21,7 +19,30 @@ Basic usage:
MIRIFLAGS="-Zmiri-genmc" cargo miri run
```
<!-- FIXME(genmc): explain options. -->
Note that `cargo miri test` in GenMC mode is currently not supported.
### Supported Parameters
- `-Zmiri-genmc`: Enable GenMC mode (not required if any other GenMC options are used).
- `-Zmiri-genmc-estimate`: This enables estimation of the concurrent execution space and verification time, before running the full verification. This should help users detect when their program is too complex to fully verify in a reasonable time. This will explore enough executions to make a good estimation, but at least 10 and at most `estimation-max` executions.
- `-Zmiri-genmc-estimation-max={MAX_ITERATIONS}`: Set the maximum number of executions that will be explored during estimation (default: 1000).
- `-Zmiri-genmc-print-exec-graphs={none,explored,blocked,all}`: Make GenMC print the execution graph of the program after every explored, every blocked, or after every execution (default: None).
- `-Zmiri-genmc-print-exec-graphs`: Shorthand for suffix `=explored`.
- `-Zmiri-genmc-print-genmc-output`: Print the output that GenMC provides. NOTE: this output is quite verbose and the events in the printed execution graph are hard to map back to the Rust code location they originate from.
- `-Zmiri-genmc-log=LOG_LEVEL`: Change the log level for GenMC. Default: `warning`.
- `quiet`: Disable logging.
- `error`: Print errors.
- `warning`: Print errors and warnings.
- `tip`: Print errors, warnings and tips.
- If Miri is built with debug assertions, there are additional log levels available (downgraded to `tip` without debug assertions):
- `debug1`: Print revisits considered by GenMC.
- `debug2`: Print the execution graph after every memory access.
- `debug3`: Print reads-from values considered by GenMC.
- `-Zmiri-genmc-verbose`: Show more information, such as estimated number of executions, and time taken for verification.
#### Regular Miri parameters useful for GenMC mode
- `-Zmiri-disable-weak-memory-emulation`: Disable any weak memory effects (effectively upgrading all atomic orderings in the program to `SeqCst`). This option may reduce the number of explored program executions, but any bugs related to weak memory effects will be missed. This option can help determine if an error is caused by weak memory effects (i.e., if it disappears with this option enabled).
<!-- FIXME(genmc): explain Miri-GenMC specific functions. -->
@ -57,6 +78,15 @@ The process for obtaining them is as follows:
If you place this directory inside the Miri folder, it is recommended to call it `genmc-src` as that tells `./miri fmt` to avoid
formatting the Rust files inside that folder.
### Formatting the C++ code
For formatting the C++ code we provide a `.clang-format` file in the `genmc-sys` directory.
With `clang-format` installed, run this command to format the c++ files (replace the `-i` with `--dry-run` to just see the changes.):
```
find ./genmc-sys/cpp/ -name "*.cpp" -o -name "*.hpp" | xargs clang-format --style=file:"./genmc-sys/.clang-format" -i
```
NOTE: this is currently not done automatically on pull requests to Miri.
<!-- FIXME(genmc): explain how submitting code to GenMC should be handled. -->
<!-- FIXME(genmc): explain development. -->

52
src/tools/miri/genmc-sys/.clang-format Normal file
View file

@ -0,0 +1,52 @@
# .clang-format
BasedOnStyle: LLVM
Standard: c++20
ColumnLimit: 100
AllowShortFunctionsOnASingleLine: Empty
AllowShortIfStatementsOnASingleLine: false
AllowShortLoopsOnASingleLine: false
AllowShortBlocksOnASingleLine: Empty
BreakBeforeBraces: Attach
BinPackArguments: false
BinPackParameters: false
AllowAllParametersOfDeclarationOnNextLine: false
AlwaysBreakAfterReturnType: None
# Force parameters to break and align
AlignAfterOpenBracket: BlockIndent
AllowAllArgumentsOnNextLine: false
# Spacing around braces and parentheses
SpaceBeforeCtorInitializerColon: true
SpaceBeforeInheritanceColon: true
SpaceInEmptyBlock: false
SpacesInContainerLiterals: true
SpacesInParensOptions:
InCStyleCasts: false
InConditionalStatements: false
InEmptyParentheses: false
Other: false
SpacesInSquareBrackets: false
# Brace spacing for initializers
Cpp11BracedListStyle: false
SpaceBeforeCpp11BracedList: true
# Import grouping: group standard, external, and project includes.
IncludeBlocks: Regroup
SortIncludes: true
# Granularity: sort includes per module/file.
IncludeIsMainRegex: '([-_](test|unittest))?$'
# Miscellaneous
SpaceAfterCStyleCast: true
SpaceBeforeParens: ControlStatements
PointerAlignment: Left
IndentCaseLabels: true
IndentWidth: 4
TabWidth: 4
UseTab: Never

8
src/tools/miri/genmc-sys/Cargo.toml
View file

@ -1,17 +1,15 @@
[package]
authors = ["Miri Team"]
# The parts in this repo are MIT OR Apache-2.0, but we are linking in
# code from https://github.com/MPI-SWS/genmc which is GPL-3.0-or-later.
license = "(MIT OR Apache-2.0) AND GPL-3.0-or-later"
license = "MIT OR Apache-2.0"
name = "genmc-sys"
version = "0.1.0"
edition = "2024"
[dependencies]
cxx = { version = "1.0.160", features = ["c++20"] }
cxx = { version = "1.0.173", features = ["c++20"] }
[build-dependencies]
cc = "1.2.16"
cmake = "0.1.54"
git2 = { version = "0.20.2", default-features = false, features = ["https"] }
cxx-build = { version = "1.0.160", features = ["parallel"] }
cxx-build = { version = "1.0.173", features = ["parallel"] }

75
src/tools/miri/genmc-sys/build.rs
View file

@ -26,17 +26,27 @@ mod downloading {
use super::GENMC_DOWNLOAD_PATH;
/// The GenMC repository the we get our commit from.
pub(crate) const GENMC_GITHUB_URL: &str = "https://github.com/MPI-SWS/genmc.git";
pub(crate) const GENMC_GITHUB_URL: &str = "https://gitlab.inf.ethz.ch/public-plf/genmc.git";
/// The GenMC commit we depend on. It must be available on the specified GenMC repository.
pub(crate) const GENMC_COMMIT: &str = "3438dd2c1202cd4a47ed7881d099abf23e4167ab";
pub(crate) const GENMC_COMMIT: &str = "af9cc9ccd5d412b16defc35dbf36571c63a19c76";
pub(crate) fn download_genmc() -> PathBuf {
/// Ensure that a local GenMC repo is present and set to the correct commit.
/// Return the path of the GenMC repo and whether the checked out commit was changed.
pub(crate) fn download_genmc() -> (PathBuf, bool) {
let Ok(genmc_download_path) = PathBuf::from_str(GENMC_DOWNLOAD_PATH);
let commit_oid = Oid::from_str(GENMC_COMMIT).expect("Commit should be valid.");
match Repository::open(&genmc_download_path) {
Ok(repo) => {
assert_repo_unmodified(&repo);
if let Ok(head) = repo.head()
&& let Ok(head_commit) = head.peel_to_commit()
&& head_commit.id() == commit_oid
{
// Fast path: The expected commit is already checked out.
return (genmc_download_path, false);
}
// Check if the local repository already contains the commit we need, download it otherwise.
let commit = update_local_repo(&repo, commit_oid);
checkout_commit(&repo, &commit);
}
@ -51,7 +61,7 @@ mod downloading {
}
};
genmc_download_path
(genmc_download_path, true)
}
fn get_remote(repo: &Repository) -> Remote<'_> {
@ -71,7 +81,8 @@ mod downloading {
// Update remote URL.
println!(
"cargo::warning=GenMC repository remote URL has changed from '{remote_url:?}' to '{GENMC_GITHUB_URL}'"
"cargo::warning=GenMC repository remote URL has changed from '{}' to '{GENMC_GITHUB_URL}'",
remote_url.unwrap_or_default()
);
repo.remote_set_url("origin", GENMC_GITHUB_URL)
.expect("cannot rename url of remote 'origin'");
@ -175,19 +186,25 @@ fn link_to_llvm(config_file: &Path) -> (String, String) {
}
/// Build the GenMC model checker library and the Rust-C++ interop library with cxx.rs
fn compile_cpp_dependencies(genmc_path: &Path) {
fn compile_cpp_dependencies(genmc_path: &Path, always_configure: bool) {
// Give each step a separate build directory to prevent interference.
let out_dir = PathBuf::from(std::env::var("OUT_DIR").as_deref().unwrap());
let genmc_build_dir = out_dir.join("genmc");
let interface_build_dir = out_dir.join("miri_genmc");
// Part 1:
// Compile the GenMC library using cmake.
let cmakelists_path = genmc_path.join("CMakeLists.txt");
// FIXME(genmc,cargo): Switch to using `CARGO_CFG_DEBUG_ASSERTIONS` once https://github.com/rust-lang/cargo/issues/15760 is completed.
// Enable/disable additional debug checks, prints and options for GenMC, based on the Rust profile (debug/release)
let enable_genmc_debug = matches!(std::env::var("PROFILE").as_deref().unwrap(), "debug");
let mut config = cmake::Config::new(cmakelists_path);
config.profile(GENMC_CMAKE_PROFILE);
config.define("GENMC_DEBUG", if enable_genmc_debug { "ON" } else { "OFF" });
let mut config = cmake::Config::new(genmc_path);
config
.always_configure(always_configure) // We force running the configure step when the GenMC commit changed.
.out_dir(genmc_build_dir)
.profile(GENMC_CMAKE_PROFILE)
.define("GENMC_DEBUG", if enable_genmc_debug { "ON" } else { "OFF" });
// The actual compilation happens here:
let genmc_install_dir = config.build();
@ -210,6 +227,13 @@ fn compile_cpp_dependencies(genmc_path: &Path) {
// These definitions are parsed into a cmake list and then printed to the config.h file, so they are ';' separated.
let definitions = llvm_definitions.split(";");
let cpp_files = [
"./cpp/src/MiriInterface/EventHandling.cpp",
"./cpp/src/MiriInterface/Exploration.cpp",
"./cpp/src/MiriInterface/Setup.cpp",
"./cpp/src/MiriInterface/ThreadManagement.cpp",
];
let mut bridge = cxx_build::bridge("src/lib.rs");
// FIXME(genmc,cmake): Remove once the GenMC debug setting is available in the config.h file.
if enable_genmc_debug {
@ -225,9 +249,9 @@ fn compile_cpp_dependencies(genmc_path: &Path) {
.std("c++23")
.include(genmc_include_dir)
.include(llvm_include_dirs)
.include("./src_cpp")
.file("./src_cpp/MiriInterface.hpp")
.file("./src_cpp/MiriInterface.cpp")
.include("./cpp/include")
.files(&cpp_files)
.out_dir(interface_build_dir)
.compile("genmc_interop");
// Link the Rust-C++ interface library generated by cxx_build:
@ -235,15 +259,9 @@ fn compile_cpp_dependencies(genmc_path: &Path) {
}
fn main() {
// Make sure we don't accidentally distribute a binary with GPL code.
if option_env!("RUSTC_STAGE").is_some() {
panic!(
"genmc should not be enabled in the rustc workspace since it includes a GPL dependency"
);
}
// Select which path to use for the GenMC repo:
let genmc_path = if let Ok(genmc_src_path) = std::env::var("GENMC_SRC_PATH") {
let (genmc_path, always_configure) = if let Some(genmc_src_path) = option_env!("GENMC_SRC_PATH")
{
let genmc_src_path =
PathBuf::from_str(&genmc_src_path).expect("GENMC_SRC_PATH should contain a valid path");
assert!(
@ -251,13 +269,20 @@ fn main() {
"GENMC_SRC_PATH={} does not exist!",
genmc_src_path.display()
);
genmc_src_path
// Rebuild files in the given path change.
println!("cargo::rerun-if-changed={}", genmc_src_path.display());
// We disable `always_configure` when working with a local repository,
// since it increases compile times when working on `genmc-sys`.
(genmc_src_path, false)
} else {
// Download GenMC if required and ensure that the correct commit is checked out.
// If anything changed in the downloaded repository (e.g., the commit),
// we set `always_configure` to ensure there are no weird configs from previous builds.
downloading::download_genmc()
};
// Build all required components:
compile_cpp_dependencies(&genmc_path);
compile_cpp_dependencies(&genmc_path, always_configure);
// Only rebuild if anything changes:
// Note that we don't add the downloaded GenMC repo, since that should never be modified
@ -265,5 +290,5 @@ fn main() {
// cloned (since cargo detects that as a file modification).
println!("cargo::rerun-if-changed={RUST_CXX_BRIDGE_FILE_PATH}");
println!("cargo::rerun-if-changed=./src");
println!("cargo::rerun-if-changed=./src_cpp");
println!("cargo::rerun-if-changed=./cpp");
}

345
src/tools/miri/genmc-sys/cpp/include/MiriInterface.hpp Normal file
View file

@ -0,0 +1,345 @@
#ifndef GENMC_MIRI_INTERFACE_HPP
#define GENMC_MIRI_INTERFACE_HPP
// CXX.rs generated headers:
#include "rust/cxx.h"
// GenMC generated headers:
#include "config.h"
// Miri `genmc-sys/src_cpp` headers:
#include "ResultHandling.hpp"
// GenMC headers:
#include "ExecutionGraph/EventLabel.hpp"
#include "Static/ModuleID.hpp"
#include "Support/MemOrdering.hpp"
#include "Support/RMWOps.hpp"
#include "Verification/Config.hpp"
#include "Verification/GenMCDriver.hpp"
// C++ headers:
#include <cstdint>
#include <format>
#include <iomanip>
#include <memory>
/**** Types available to both Rust and C++ ****/
struct GenmcParams;
enum class LogLevel : std::uint8_t;
struct GenmcScalar;
struct SchedulingResult;
struct EstimationResult;
struct LoadResult;
struct StoreResult;
struct ReadModifyWriteResult;
struct CompareExchangeResult;
// GenMC uses `int` for its thread IDs.
using ThreadId = int;
/// Set the log level for GenMC.
///
/// # Safety
///
/// This function is not thread safe, since it writes to the global, mutable, non-atomic `logLevel`
/// variable. Any GenMC function may read from `logLevel` unsynchronized.
/// The safest way to use this function is to set the log level exactly once before first calling
/// `create_handle`.
/// Never calling this function is safe, GenMC will fall back to its default log level.
/* unsafe */ void set_log_level_raw(LogLevel log_level);
struct MiriGenmcShim : private GenMCDriver {
public:
MiriGenmcShim(std::shared_ptr<const Config> conf, Mode mode /* = VerificationMode{} */)
: GenMCDriver(std::move(conf), nullptr, mode) {}
/// Create a new `MiriGenmcShim`, which wraps a `GenMCDriver`.
///
/// # Safety
///
/// This function is marked as unsafe since the `logLevel` global variable is non-atomic.
/// This function should not be called in an unsynchronized way with `set_log_level_raw`,
/// since this function and any methods on the returned `MiriGenmcShim` may read the
/// `logLevel`, causing a data race. The safest way to use these functions is to call
/// `set_log_level_raw` once, and only then start creating handles. There should not be any
/// other (safe) way to create a `MiriGenmcShim`.
/* unsafe */ static auto create_handle(const GenmcParams& params, bool estimation_mode)
-> std::unique_ptr<MiriGenmcShim>;
virtual ~MiriGenmcShim() {}
/**** Execution start/end handling ****/
// This function must be called at the start of any execution, before any events are
// reported to GenMC.
void handle_execution_start();
// This function must be called at the end of any execution, even if an error was found
// during the execution.
// Returns `null`, or a string containing an error message if an error occured.
std::unique_ptr<std::string> handle_execution_end();
/***** Functions for handling events encountered during program execution. *****/
/**** Memory access handling ****/
[[nodiscard]] LoadResult handle_load(
ThreadId thread_id,
uint64_t address,
uint64_t size,
MemOrdering ord,
GenmcScalar old_val
);
[[nodiscard]] ReadModifyWriteResult handle_read_modify_write(
ThreadId thread_id,
uint64_t address,
uint64_t size,
RMWBinOp rmw_op,
MemOrdering ordering,
GenmcScalar rhs_value,
GenmcScalar old_val
);
[[nodiscard]] CompareExchangeResult handle_compare_exchange(
ThreadId thread_id,
uint64_t address,
uint64_t size,
GenmcScalar expected_value,
GenmcScalar new_value,
GenmcScalar old_val,
MemOrdering success_ordering,
MemOrdering fail_load_ordering,
bool can_fail_spuriously
);
[[nodiscard]] StoreResult handle_store(
ThreadId thread_id,
uint64_t address,
uint64_t size,
GenmcScalar value,
GenmcScalar old_val,
MemOrdering ord
);
void handle_fence(ThreadId thread_id, MemOrdering ord);
/**** Memory (de)allocation ****/
auto handle_malloc(ThreadId thread_id, uint64_t size, uint64_t alignment) -> uint64_t;
void handle_free(ThreadId thread_id, uint64_t address);
/**** Thread management ****/
void handle_thread_create(ThreadId thread_id, ThreadId parent_id);
void handle_thread_join(ThreadId thread_id, ThreadId child_id);
void handle_thread_finish(ThreadId thread_id, uint64_t ret_val);
void handle_thread_kill(ThreadId thread_id);
/***** Exploration related functionality *****/
/** Ask the GenMC scheduler for a new thread to schedule and return whether the execution is
* finished, blocked, or can continue.
* Updates the next instruction kind for the given thread id. */
auto schedule_next(const int curr_thread_id, const ActionKind curr_thread_next_instr_kind)
-> SchedulingResult;
/**
* Check whether there are more executions to explore.
* If there are more executions, this method prepares for the next execution and returns
* `true`. Returns true if there are no more executions to explore. */
auto is_exploration_done() -> bool {
return GenMCDriver::done();
}
/**** Result querying functionality. ****/
// NOTE: We don't want to share the `VerificationResult` type with the Rust side, since it
// is very large, uses features that CXX.rs doesn't support and may change as GenMC changes.
// Instead, we only use the result on the C++ side, and only expose these getter function to
// the Rust side.
// Note that CXX.rs doesn't support returning a C++ string to Rust by value,
// it must be behind an indirection like a `unique_ptr` (tested with CXX 1.0.170).
/// Get the number of blocked executions encountered by GenMC (cast into a fixed with
/// integer)
auto get_blocked_execution_count() const -> uint64_t {
return static_cast<uint64_t>(getResult().exploredBlocked);
}
/// Get the number of executions explored by GenMC (cast into a fixed with integer)
auto get_explored_execution_count() const -> uint64_t {
return static_cast<uint64_t>(getResult().explored);
}
/// Get all messages that GenMC produced (errors, warnings), combined into one string.
auto get_result_message() const -> std::unique_ptr<std::string> {
return std::make_unique<std::string>(getResult().message);
}
/// If an error occurred, return a string describing the error, otherwise, return `nullptr`.
auto get_error_string() const -> std::unique_ptr<std::string> {
const auto& result = GenMCDriver::getResult();
if (result.status.has_value())
return format_error(result.status.value());
return nullptr;
}
/**** Printing and estimation mode functionality. ****/
/// Get the results of a run in estimation mode.
auto get_estimation_results() const -> EstimationResult;
private:
/** Increment the event index in the given thread by 1 and return the new event. */
[[nodiscard]] inline auto inc_pos(ThreadId tid) -> Event {
ERROR_ON(tid >= threads_action_.size(), "ThreadId out of bounds");
return ++threads_action_[tid].event;
}
/** Decrement the event index in the given thread by 1 and return the new event. */
inline auto dec_pos(ThreadId tid) -> Event {
ERROR_ON(tid >= threads_action_.size(), "ThreadId out of bounds");
return --threads_action_[tid].event;
}
/**
* Helper function for loads that need to reset the event counter when no value is returned.
* Same syntax as `GenMCDriver::handleLoad`, but this takes a thread id instead of an Event.
* Automatically calls `inc_pos` and `dec_pos` where needed for the given thread.
*/
template <EventLabel::EventLabelKind k, typename... Ts>
auto handle_load_reset_if_none(ThreadId tid, Ts&&... params) -> HandleResult<SVal> {
const auto pos = inc_pos(tid);
const auto ret = GenMCDriver::handleLoad<k>(pos, std::forward<Ts>(params)...);
// If we didn't get a value, we have to reset the index of the current thread.
if (!std::holds_alternative<SVal>(ret)) {
dec_pos(tid);
}
return ret;
}
/**
* GenMC uses the term `Action` to refer to a struct of:
* - `ActionKind`, storing whether the next instruction in a thread may be a load
* - `Event`, storing the most recent event index added for a thread
*
* Here we store the "action" for each thread. In particular we use this to assign event
* indices, since GenMC expects us to do that.
*/
std::vector<Action> threads_action_;
};
/// Get the bit mask that GenMC expects for global memory allocations.
/// FIXME(genmc): currently we use `get_global_alloc_static_mask()` to ensure the
/// `SAddr::staticMask` constant is consistent between Miri and GenMC, but if
/// https://github.com/dtolnay/cxx/issues/1051 is fixed we could share the constant
/// directly.
constexpr auto get_global_alloc_static_mask() -> uint64_t {
return SAddr::staticMask;
}
// CXX.rs generated headers:
// NOTE: this must be included *after* `MiriGenmcShim` and all the other types we use are defined,
// otherwise there will be compilation errors due to missing definitions.
#include "genmc-sys/src/lib.rs.h"
/**** Result handling ****/
// NOTE: these must come after the cxx_bridge generated code, since that contains the actual
// definitions of these types.
namespace GenmcScalarExt {
inline GenmcScalar uninit() {
return GenmcScalar {
.value = 0,
.is_init = false,
};
}
inline GenmcScalar from_sval(SVal sval) {
return GenmcScalar {
.value = sval.get(),
.is_init = true,
};
}
inline SVal to_sval(GenmcScalar scalar) {
ERROR_ON(!scalar.is_init, "Cannot convert an uninitialized `GenmcScalar` into an `SVal`\n");
return SVal(scalar.value);
}
} // namespace GenmcScalarExt
namespace LoadResultExt {
inline LoadResult no_value() {
return LoadResult {
.error = std::unique_ptr<std::string>(nullptr),
.has_value = false,
.read_value = GenmcScalarExt::uninit(),
};
}
inline LoadResult from_value(SVal read_value) {
return LoadResult { .error = std::unique_ptr<std::string>(nullptr),
.has_value = true,
.read_value = GenmcScalarExt::from_sval(read_value) };
}
inline LoadResult from_error(std::unique_ptr<std::string> error) {
return LoadResult { .error = std::move(error),
.has_value = false,
.read_value = GenmcScalarExt::uninit() };
}
} // namespace LoadResultExt
namespace StoreResultExt {
inline StoreResult ok(bool is_coherence_order_maximal_write) {
return StoreResult { /* error: */ std::unique_ptr<std::string>(nullptr),
is_coherence_order_maximal_write };
}
inline StoreResult from_error(std::unique_ptr<std::string> error) {
return StoreResult { .error = std::move(error), .is_coherence_order_maximal_write = false };
}
} // namespace StoreResultExt
namespace ReadModifyWriteResultExt {
inline ReadModifyWriteResult
ok(SVal old_value, SVal new_value, bool is_coherence_order_maximal_write) {
return ReadModifyWriteResult { .error = std::unique_ptr<std::string>(nullptr),
.old_value = GenmcScalarExt::from_sval(old_value),
.new_value = GenmcScalarExt::from_sval(new_value),
.is_coherence_order_maximal_write =
is_coherence_order_maximal_write };
}
inline ReadModifyWriteResult from_error(std::unique_ptr<std::string> error) {
return ReadModifyWriteResult { .error = std::move(error),
.old_value = GenmcScalarExt::uninit(),
.new_value = GenmcScalarExt::uninit(),
.is_coherence_order_maximal_write = false };
}
} // namespace ReadModifyWriteResultExt
namespace CompareExchangeResultExt {
inline CompareExchangeResult success(SVal old_value, bool is_coherence_order_maximal_write) {
return CompareExchangeResult { .error = nullptr,
.old_value = GenmcScalarExt::from_sval(old_value),
.is_success = true,
.is_coherence_order_maximal_write =
is_coherence_order_maximal_write };
}
inline CompareExchangeResult failure(SVal old_value) {
return CompareExchangeResult { .error = nullptr,
.old_value = GenmcScalarExt::from_sval(old_value),
.is_success = false,
.is_coherence_order_maximal_write = false };
}
inline CompareExchangeResult from_error(std::unique_ptr<std::string> error) {
return CompareExchangeResult { .error = std::move(error),
.old_value = GenmcScalarExt::uninit(),
.is_success = false,
.is_coherence_order_maximal_write = false };
}
} // namespace CompareExchangeResultExt
#endif /* GENMC_MIRI_INTERFACE_HPP */

21
src/tools/miri/genmc-sys/cpp/include/ResultHandling.hpp Normal file
View file

@ -0,0 +1,21 @@
#ifndef GENMC_RESULT_HANDLING_HPP
#define GENMC_RESULT_HANDLING_HPP
// CXX.rs generated headers:
#include "rust/cxx.h"
// GenMC headers:
#include "Verification/VerificationError.hpp"
#include <string>
/** Information about an error, formatted as a string to avoid having to share an error enum and
* printing functionality with the Rust side. */
static auto format_error(VerificationError err) -> std::unique_ptr<std::string> {
auto buf = std::string();
auto s = llvm::raw_string_ostream(buf);
s << err;
return std::make_unique<std::string>(s.str());
}
#endif /* GENMC_RESULT_HANDLING_HPP */

251
src/tools/miri/genmc-sys/cpp/src/MiriInterface/EventHandling.cpp Normal file
View file

@ -0,0 +1,251 @@
/** This file contains functionality related to handling events encountered
* during an execution, such as loads, stores or memory (de)allocation. */
#include "MiriInterface.hpp"
// CXX.rs generated headers:
#include "genmc-sys/src/lib.rs.h"
// GenMC headers:
#include "ADT/value_ptr.hpp"
#include "ExecutionGraph/EventLabel.hpp"
#include "ExecutionGraph/LoadAnnotation.hpp"
#include "Runtime/InterpreterEnumAPI.hpp"
#include "Static/ModuleID.hpp"
#include "Support/ASize.hpp"
#include "Support/Error.hpp"
#include "Support/Logger.hpp"
#include "Support/MemAccess.hpp"
#include "Support/RMWOps.hpp"
#include "Support/SAddr.hpp"
#include "Support/SVal.hpp"
#include "Support/ThreadInfo.hpp"
#include "Support/Verbosity.hpp"
#include "Verification/GenMCDriver.hpp"
#include "Verification/MemoryModel.hpp"
// C++ headers:
#include <cstddef>
#include <cstdint>
#include <memory>
#include <utility>
/**** Memory access handling ****/
[[nodiscard]] auto MiriGenmcShim::handle_load(
ThreadId thread_id,
uint64_t address,
uint64_t size,
MemOrdering ord,
GenmcScalar old_val
) -> LoadResult {
// `type` is only used for printing.
const auto type = AType::Unsigned;
const auto ret = handle_load_reset_if_none<EventLabel::EventLabelKind::Read>(
thread_id,
ord,
SAddr(address),
ASize(size),
type
);
if (const auto* err = std::get_if<VerificationError>(&ret))
return LoadResultExt::from_error(format_error(*err));
const auto* ret_val = std::get_if<SVal>(&ret);
if (ret_val == nullptr)
ERROR("Unimplemented: load returned unexpected result.");
return LoadResultExt::from_value(*ret_val);
}
[[nodiscard]] auto MiriGenmcShim::handle_store(
ThreadId thread_id,
uint64_t address,
uint64_t size,
GenmcScalar value,
GenmcScalar old_val,
MemOrdering ord
) -> StoreResult {
const auto pos = inc_pos(thread_id);
const auto ret = GenMCDriver::handleStore<EventLabel::EventLabelKind::Write>(
pos,
ord,
SAddr(address),
ASize(size),
/* type */ AType::Unsigned, // `type` is only used for printing.
GenmcScalarExt::to_sval(value),
EventDeps()
);
if (const auto* err = std::get_if<VerificationError>(&ret))
return StoreResultExt::from_error(format_error(*err));
if (!std::holds_alternative<std::monostate>(ret))
ERROR("store returned unexpected result");
// FIXME(genmc,mixed-accesses): Use the value that GenMC returns from handleStore (once
// available).
const auto& g = getExec().getGraph();
return StoreResultExt::ok(
/* is_coherence_order_maximal_write */ g.co_max(SAddr(address))->getPos() == pos
);
}
void MiriGenmcShim::handle_fence(ThreadId thread_id, MemOrdering ord) {
const auto pos = inc_pos(thread_id);
GenMCDriver::handleFence(pos, ord, EventDeps());
}
[[nodiscard]] auto MiriGenmcShim::handle_read_modify_write(
ThreadId thread_id,
uint64_t address,
uint64_t size,
RMWBinOp rmw_op,
MemOrdering ordering,
GenmcScalar rhs_value,
GenmcScalar old_val
) -> ReadModifyWriteResult {
// NOTE: Both the store and load events should get the same `ordering`, it should not be split
// into a load and a store component. This means we can have for example `AcqRel` loads and
// stores, but this is intended for RMW operations.
// Somewhat confusingly, the GenMC term for RMW read/write labels is
// `FaiRead` and `FaiWrite`.
const auto load_ret = handle_load_reset_if_none<EventLabel::EventLabelKind::FaiRead>(
thread_id,
ordering,
SAddr(address),
ASize(size),
AType::Unsigned, // The type is only used for printing.
rmw_op,
GenmcScalarExt::to_sval(rhs_value),
EventDeps()
);
if (const auto* err = std::get_if<VerificationError>(&load_ret))
return ReadModifyWriteResultExt::from_error(format_error(*err));
const auto* ret_val = std::get_if<SVal>(&load_ret);
if (nullptr == ret_val) {
ERROR("Unimplemented: read-modify-write returned unexpected result.");
}
const auto read_old_val = *ret_val;
const auto new_value =
executeRMWBinOp(read_old_val, GenmcScalarExt::to_sval(rhs_value), size, rmw_op);
const auto storePos = inc_pos(thread_id);
const auto store_ret = GenMCDriver::handleStore<EventLabel::EventLabelKind::FaiWrite>(
storePos,
ordering,
SAddr(address),
ASize(size),
AType::Unsigned, // The type is only used for printing.
new_value
);
if (const auto* err = std::get_if<VerificationError>(&store_ret))
return ReadModifyWriteResultExt::from_error(format_error(*err));
const auto* store_ret_val = std::get_if<std::monostate>(&store_ret);
ERROR_ON(nullptr == store_ret_val, "Unimplemented: RMW store returned unexpected result.");
// FIXME(genmc,mixed-accesses): Use the value that GenMC returns from handleStore (once
// available).
const auto& g = getExec().getGraph();
return ReadModifyWriteResultExt::ok(
/* old_value: */ read_old_val,
new_value,
/* is_coherence_order_maximal_write */ g.co_max(SAddr(address))->getPos() == storePos
);
}
[[nodiscard]] auto MiriGenmcShim::handle_compare_exchange(
ThreadId thread_id,
uint64_t address,
uint64_t size,
GenmcScalar expected_value,
GenmcScalar new_value,
GenmcScalar old_val,
MemOrdering success_ordering,
MemOrdering fail_load_ordering,
bool can_fail_spuriously
) -> CompareExchangeResult {
// NOTE: Both the store and load events should get the same `ordering`, it should not be split
// into a load and a store component. This means we can have for example `AcqRel` loads and
// stores, but this is intended for CAS operations.
// FIXME(GenMC): properly handle failure memory ordering.
auto expectedVal = GenmcScalarExt::to_sval(expected_value);
auto new_val = GenmcScalarExt::to_sval(new_value);
const auto load_ret = handle_load_reset_if_none<EventLabel::EventLabelKind::CasRead>(
thread_id,
success_ordering,
SAddr(address),
ASize(size),
AType::Unsigned, // The type is only used for printing.
expectedVal,
new_val
);
if (const auto* err = std::get_if<VerificationError>(&load_ret))
return CompareExchangeResultExt::from_error(format_error(*err));
const auto* ret_val = std::get_if<SVal>(&load_ret);
ERROR_ON(nullptr == ret_val, "Unimplemented: load returned unexpected result.");
const auto read_old_val = *ret_val;
if (read_old_val != expectedVal)
return CompareExchangeResultExt::failure(read_old_val);
// FIXME(GenMC): Add support for modelling spurious failures.
const auto storePos = inc_pos(thread_id);
const auto store_ret = GenMCDriver::handleStore<EventLabel::EventLabelKind::CasWrite>(
storePos,
success_ordering,
SAddr(address),
ASize(size),
AType::Unsigned, // The type is only used for printing.
new_val
);
if (const auto* err = std::get_if<VerificationError>(&store_ret))
return CompareExchangeResultExt::from_error(format_error(*err));
const auto* store_ret_val = std::get_if<std::monostate>(&store_ret);
ERROR_ON(
nullptr == store_ret_val,
"Unimplemented: compare-exchange store returned unexpected result."
);
// FIXME(genmc,mixed-accesses): Use the value that GenMC returns from handleStore (once
// available).
const auto& g = getExec().getGraph();
return CompareExchangeResultExt::success(
read_old_val,
/* is_coherence_order_maximal_write */ g.co_max(SAddr(address))->getPos() == storePos
);
}
/**** Memory (de)allocation ****/
auto MiriGenmcShim::handle_malloc(ThreadId thread_id, uint64_t size, uint64_t alignment)
-> uint64_t {
const auto pos = inc_pos(thread_id);
// These are only used for printing and features Miri-GenMC doesn't support (yet).
const auto storage_duration = StorageDuration::SD_Heap;
// Volatile, as opposed to "persistent" (i.e., non-volatile memory that persists over reboots)
const auto storage_type = StorageType::ST_Volatile;
const auto address_space = AddressSpace::AS_User;
const SVal ret_val = GenMCDriver::handleMalloc(
pos,
size,
alignment,
storage_duration,
storage_type,
address_space,
EventDeps()
);
return ret_val.get();
}
void MiriGenmcShim::handle_free(ThreadId thread_id, uint64_t address) {
const auto pos = inc_pos(thread_id);
GenMCDriver::handleFree(pos, SAddr(address), EventDeps());
// FIXME(genmc): add error handling once GenMC returns errors from `handleFree`
}

56
src/tools/miri/genmc-sys/cpp/src/MiriInterface/Exploration.cpp Normal file
View file

@ -0,0 +1,56 @@
/** This file contains functionality related to exploration, such as scheduling. */
#include "MiriInterface.hpp"
// CXX.rs generated headers:
#include "genmc-sys/src/lib.rs.h"
// GenMC headers:
#include "Support/Error.hpp"
#include "Support/Verbosity.hpp"
// C++ headers:
#include <cmath>
#include <cstdint>
#include <limits>
auto MiriGenmcShim::schedule_next(
const int curr_thread_id,
const ActionKind curr_thread_next_instr_kind
) -> SchedulingResult {
// The current thread is the only one where the `kind` could have changed since we last made
// a scheduling decision.
threads_action_[curr_thread_id].kind = curr_thread_next_instr_kind;
if (const auto result = GenMCDriver::scheduleNext(threads_action_))
return SchedulingResult { ExecutionState::Ok, static_cast<int32_t>(result.value()) };
if (GenMCDriver::isExecutionBlocked())
return SchedulingResult { ExecutionState::Blocked, 0 };
return SchedulingResult { ExecutionState::Finished, 0 };
}
/**** Execution start/end handling ****/
void MiriGenmcShim::handle_execution_start() {
threads_action_.clear();
threads_action_.push_back(Action(ActionKind::Load, Event::getInit()));
GenMCDriver::handleExecutionStart();
}
auto MiriGenmcShim::handle_execution_end() -> std::unique_ptr<std::string> {
// FIXME(genmc): add error handling once GenMC returns an error here.
GenMCDriver::handleExecutionEnd();
return {};
}
/**** Estimation mode result ****/
auto MiriGenmcShim::get_estimation_results() const -> EstimationResult {
const auto& res = getResult();
return EstimationResult {
.mean = static_cast<double>(res.estimationMean),
.sd = static_cast<double>(std::sqrt(res.estimationVariance)),
.explored_execs = static_cast<uint64_t>(res.explored),
.blocked_execs = static_cast<uint64_t>(res.exploredBlocked),
};
}

197
src/tools/miri/genmc-sys/cpp/src/MiriInterface/Setup.cpp Normal file
View file

@ -0,0 +1,197 @@
/** This file contains functionality related to creation of the GenMCDriver,
* including translating settings set by Miri. */
#include "MiriInterface.hpp"
// CXX.rs generated headers:
#include "genmc-sys/src/lib.rs.h"
// GenMC headers:
#include "Support/Error.hpp"
#include "Support/Verbosity.hpp"
#include "Verification/InterpreterCallbacks.hpp"
// C++ headers:
#include <cstdint>
/**
* Translate the Miri-GenMC `LogLevel` to the GenMC `VerbosityLevel`.
* Downgrade any debug options to `Tip` if `ENABLE_GENMC_DEBUG` is not enabled.
*/
static auto to_genmc_verbosity_level(const LogLevel log_level) -> VerbosityLevel {
switch (log_level) {
case LogLevel::Quiet:
return VerbosityLevel::Quiet;
case LogLevel::Error:
return VerbosityLevel::Error;
case LogLevel::Warning:
return VerbosityLevel::Warning;
case LogLevel::Tip:
return VerbosityLevel::Tip;
#ifdef ENABLE_GENMC_DEBUG
case LogLevel::Debug1Revisits:
return VerbosityLevel::Debug1;
case LogLevel::Debug2MemoryAccesses:
return VerbosityLevel::Debug2;
case LogLevel::Debug3ReadsFrom:
return VerbosityLevel::Debug3;
#else
// Downgrade to `Tip` if the debug levels are not available.
case LogLevel::Debug1Revisits:
case LogLevel::Debug2MemoryAccesses:
case LogLevel::Debug3ReadsFrom:
return VerbosityLevel::Tip;
#endif
default:
WARN_ONCE(
"unknown-log-level",
"Unknown `LogLevel`, defaulting to `VerbosityLevel::Tip`."
);
return VerbosityLevel::Tip;
}
}
/* unsafe */ void set_log_level_raw(LogLevel log_level) {
// The `logLevel` is a static, non-atomic variable.
// It should never be changed if `MiriGenmcShim` still exists, since any of its methods may read
// the `logLevel`, otherwise it may cause data races.
logLevel = to_genmc_verbosity_level(log_level);
}
/* unsafe */ auto MiriGenmcShim::create_handle(const GenmcParams& params, bool estimation_mode)
-> std::unique_ptr<MiriGenmcShim> {
auto conf = std::make_shared<Config>();
// Set whether GenMC should print execution graphs after every explored/blocked execution.
conf->printExecGraphs =
(params.print_execution_graphs == ExecutiongraphPrinting::Explored ||
params.print_execution_graphs == ExecutiongraphPrinting::ExploredAndBlocked);
conf->printBlockedExecs =
(params.print_execution_graphs == ExecutiongraphPrinting::Blocked ||
params.print_execution_graphs == ExecutiongraphPrinting::ExploredAndBlocked);
// `1024` is the default value that GenMC uses.
// If any thread has at least this many events, a warning/tip will be printed.
//
// Miri produces a lot more events than GenMC, so the graph size warning triggers on almost
// all programs. The current value is large enough so the warning is not be triggered by any
// reasonable programs.
// FIXME(genmc): The emitted warning mentions features not supported by Miri ('--unroll'
// parameter).
// FIXME(genmc): A more appropriate limit should be chosen once the warning is useful for
// Miri.
conf->warnOnGraphSize = 1024 * 1024;
// We only support the `RC11` memory model for Rust, and `SC` when weak memory emulation is
// disabled.
conf->model = params.disable_weak_memory_emulation ? ModelType::SC : ModelType::RC11;
// This prints the seed that GenMC picks for randomized scheduling during estimation mode.
conf->printRandomScheduleSeed = params.print_random_schedule_seed;
// FIXME(genmc): supporting IPR requires annotations for `assume` and `spinloops`.
conf->ipr = false;
// FIXME(genmc): supporting BAM requires `Barrier` support + detecting whether return value
// of barriers are used.
conf->disableBAM = true;
// Instruction caching could help speed up verification by filling the graph from cache, if
// the list of values read by all load events in a thread have been seen before. Combined
// with not replaying completed threads, this can also reducing the amount of Mir
// interpretation required by Miri. With the current setup, this would be incorrect, since
// Miri doesn't give GenMC the actual values read by non-atomic reads.
conf->instructionCaching = false;
// Many of Miri's checks work under the assumption that threads are only executed in an
// order that could actually happen during a normal execution. Formally, this means that
// replaying an execution needs to respect the po-rf-relation of the executiongraph (po ==
// program-order, rf == reads-from). This means, any event in the graph, when replayed, must
// happen after any events that happen before it in the same graph according to the program
// code, and all (non-atomic) reads must happen after the write event they read from.
//
// Not replaying completed threads means any read event from that thread never happens in
// Miri's memory, so this would only work if there are never any non-atomic reads from any
// value written by the skipped thread.
conf->replayCompletedThreads = true;
// FIXME(genmc): implement symmetry reduction.
ERROR_ON(
params.do_symmetry_reduction,
"Symmetry reduction is currently unsupported in GenMC mode."
);
conf->symmetryReduction = params.do_symmetry_reduction;
// Set the scheduling policy. GenMC uses `WFR` for estimation mode.
// For normal verification, `WF` has the best performance and is the GenMC default.
// Other scheduling policies are used by GenMC for testing and for modes currently
// unsupported with Miri such as bounding, which uses LTR.
conf->schedulePolicy = estimation_mode ? SchedulePolicy::WFR : SchedulePolicy::WF;
// Set the min and max number of executions tested in estimation mode.
conf->estimationMin = 10; // default taken from GenMC
conf->estimationMax = params.estimation_max;
// Deviation threshold % under which estimation is deemed good enough.
conf->sdThreshold = 10; // default taken from GenMC
// Set the mode used for this driver, either estimation or verification.
const auto mode = estimation_mode ? GenMCDriver::Mode(GenMCDriver::EstimationMode {})
: GenMCDriver::Mode(GenMCDriver::VerificationMode {});
// Running Miri-GenMC without race detection is not supported.
// Disabling this option also changes the behavior of the replay scheduler to only schedule
// at atomic operations, which is required with Miri. This happens because Miri can generate
// multiple GenMC events for a single MIR terminator. Without this option, the scheduler
// might incorrectly schedule an atomic MIR terminator because the first event it creates is
// a non-atomic (e.g., `StorageLive`).
conf->disableRaceDetection = false;
// Miri can already check for unfreed memory. Also, GenMC cannot distinguish between memory
// that is allowed to leak and memory that is not.
conf->warnUnfreedMemory = false;
// FIXME(genmc,error handling): This function currently exits on error, but will return an
// error value in the future. The return value should be checked once this change is made.
checkConfig(*conf);
// Create the actual driver and Miri-GenMC communication shim.
auto driver = std::make_unique<MiriGenmcShim>(std::move(conf), mode);
// FIXME(genmc,HACK): Until a proper solution is implemented in GenMC, these callbacks will
// allow Miri to return information about global allocations and override uninitialized memory
// checks for non-atomic loads (Miri handles those without GenMC, so the error would be wrong).
auto interpreter_callbacks = InterpreterCallbacks {
// Miri already ensures that memory accesses are valid, so this check doesn't matter.
// We check that the address is static, but skip checking if it is part of an actual
// allocation.
.isStaticallyAllocated = [](SAddr addr) { return addr.isStatic(); },
// FIXME(genmc,error reporting): Once a proper a proper API for passing such information is
// implemented in GenMC, Miri should use it to improve the produced error messages.
.getStaticName = [](SAddr addr) { return "[UNKNOWN STATIC]"; },
// This function is called to get the initial value stored at the given address.
//
// From a Miri perspective, this API doesn't work very well: most memory starts out
// "uninitialized";
// only statics have an initial value. And their initial value is just a sequence of bytes,
// but GenMC
// expect this to be already split into separate atomic variables. So we return a dummy
// value.
// This value should never be visible to the interpreted program.
// GenMC does not understand uninitialized memory the same way Miri does, which may cause
// this function to be called. The returned value can be visible to Miri or the user:
// - Printing the execution graph may contain this value in place of uninitialized values.
// FIXME(genmc): NOTE: printing the execution graph is not yet implemented.
// - Non-atomic loads may return this value, but Miri ignores values of non-atomic loads.
// - Atomic loads will *not* see this value once mixed atomic-non-atomic support is added.
// Currently, atomic loads can see this value, unless initialized by an *atomic* store.
// FIXME(genmc): update this comment once mixed atomic-non-atomic support is added.
//
// FIXME(genmc): implement proper support for uninitialized memory in GenMC. Ideally, the
// initial value getter would return an `optional<SVal>`, since the memory location may be
// uninitialized.
.initValGetter = [](const AAccess& a) { return SVal(0xDEAD); },
// Miri serves non-atomic loads from its own memory and these GenMC checks are wrong in
// that case. This should no longer be required with proper mixed-size access support.
.skipUninitLoadChecks = [](MemOrdering ord) { return ord == MemOrdering::NotAtomic; },
};
driver->setInterpCallbacks(std::move(interpreter_callbacks));
return driver;
}

54
src/tools/miri/genmc-sys/cpp/src/MiriInterface/ThreadManagement.cpp Normal file
View file

@ -0,0 +1,54 @@
/** This file contains functionality related thread management (creation, finishing, join, etc.) */
#include "MiriInterface.hpp"
// CXX.rs generated headers:
#include "genmc-sys/src/lib.rs.h"
// GenMC headers:
#include "Support/Error.hpp"
#include "Support/Verbosity.hpp"
// C++ headers:
#include <cstdint>
void MiriGenmcShim::handle_thread_create(ThreadId thread_id, ThreadId parent_id) {
// NOTE: The threadCreate event happens in the parent:
const auto pos = inc_pos(parent_id);
// FIXME(genmc): for supporting symmetry reduction, these will need to be properly set:
const unsigned fun_id = 0;
const SVal arg = SVal(0);
const ThreadInfo child_info = ThreadInfo { thread_id, parent_id, fun_id, arg };
// NOTE: Default memory ordering (`Release`) used here.
const auto child_tid = GenMCDriver::handleThreadCreate(pos, child_info, EventDeps());
// Sanity check the thread id, which is the index in the `threads_action_` array.
BUG_ON(child_tid != thread_id || child_tid <= 0 || child_tid != threads_action_.size());
threads_action_.push_back(Action(ActionKind::Load, Event(child_tid, 0)));
}
void MiriGenmcShim::handle_thread_join(ThreadId thread_id, ThreadId child_id) {
// The thread join event happens in the parent.
const auto pos = inc_pos(thread_id);
// NOTE: Default memory ordering (`Acquire`) used here.
const auto ret = GenMCDriver::handleThreadJoin(pos, child_id, EventDeps());
// If the join failed, decrease the event index again:
if (!std::holds_alternative<SVal>(ret)) {
dec_pos(thread_id);
}
// NOTE: Thread return value is ignored, since Miri doesn't need it.
}
void MiriGenmcShim::handle_thread_finish(ThreadId thread_id, uint64_t ret_val) {
const auto pos = inc_pos(thread_id);
// NOTE: Default memory ordering (`Release`) used here.
GenMCDriver::handleThreadFinish(pos, SVal(ret_val));
}
void MiriGenmcShim::handle_thread_kill(ThreadId thread_id) {
const auto pos = inc_pos(thread_id);
GenMCDriver::handleThreadKill(pos);
}

436
src/tools/miri/genmc-sys/src/lib.rs
View file

@ -1,30 +1,456 @@
use std::str::FromStr;
use std::sync::OnceLock;
pub use cxx::UniquePtr;
pub use self::ffi::*;
/// Defined in "genmc/src/Support/SAddr.hpp".
/// The first bit of all global addresses must be set to `1`.
/// This means the mask, interpreted as an address, is the lower bound of where the global address space starts.
///
/// FIXME(genmc): rework this if non-64bit support is added to GenMC (the current allocation scheme only allows for 64bit addresses).
/// FIXME(genmc): currently we use `get_global_alloc_static_mask()` to ensure the constant is consistent between Miri and GenMC,
/// but if https://github.com/dtolnay/cxx/issues/1051 is fixed we could share the constant directly.
pub const GENMC_GLOBAL_ADDRESSES_MASK: u64 = 1 << 63;
/// GenMC thread ids are C++ type `int`, which is equivalent to Rust's `i32` on most platforms.
/// The main thread always has thread id 0.
pub const GENMC_MAIN_THREAD_ID: i32 = 0;
/// Changing GenMC's log level is not thread safe, so we limit it to only be set once to prevent any data races.
/// This value will be initialized when the first `MiriGenmcShim` is created.
static GENMC_LOG_LEVEL: OnceLock<LogLevel> = OnceLock::new();
// Create a new handle to the GenMC model checker.
// The first call to this function determines the log level of GenMC, any future call with a different log level will panic.
pub fn create_genmc_driver_handle(
params: &GenmcParams,
genmc_log_level: LogLevel,
do_estimation: bool,
) -> UniquePtr<MiriGenmcShim> {
// SAFETY: Only setting the GenMC log level once is guaranteed by the `OnceLock`.
// No other place calls `set_log_level_raw`, so the `logLevel` value in GenMC will not change once we initialize it once.
// All functions that use GenMC's `logLevel` can only be accessed in safe Rust through a `MiriGenmcShim`.
// There is no way to get `MiriGenmcShim` other than through `create_handle`, and we only call it *after* setting the log level, preventing any possible data races.
assert_eq!(
&genmc_log_level,
GENMC_LOG_LEVEL.get_or_init(|| {
unsafe { set_log_level_raw(genmc_log_level) };
genmc_log_level
}),
"Attempt to change the GenMC log level after it was already set"
);
unsafe { MiriGenmcShim::create_handle(params, do_estimation) }
}
impl GenmcScalar {
pub const UNINIT: Self = Self { value: 0, is_init: false };
pub const fn from_u64(value: u64) -> Self {
Self { value, is_init: true }
}
}
impl Default for GenmcParams {
fn default() -> Self {
Self {
estimation_max: 1000, // default taken from GenMC
print_random_schedule_seed: false,
do_symmetry_reduction: false,
// FIXME(GenMC): Add defaults for remaining parameters
// GenMC graphs can be quite large since Miri produces a lot of (non-atomic) events.
print_execution_graphs: ExecutiongraphPrinting::None,
disable_weak_memory_emulation: false,
}
}
}
impl Default for LogLevel {
fn default() -> Self {
// FIXME(genmc): set `Tip` by default once the GenMC tips are relevant to Miri.
Self::Warning
}
}
impl FromStr for SchedulePolicy {
type Err = &'static str;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(match s {
"wf" => SchedulePolicy::WF,
"wfr" => SchedulePolicy::WFR,
"arbitrary" | "random" => SchedulePolicy::Arbitrary,
"ltr" => SchedulePolicy::LTR,
_ => return Err("invalid scheduling policy"),
})
}
}
impl FromStr for LogLevel {
type Err = &'static str;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Ok(match s {
"quiet" => LogLevel::Quiet,
"error" => LogLevel::Error,
"warning" => LogLevel::Warning,
"tip" => LogLevel::Tip,
"debug1" => LogLevel::Debug1Revisits,
"debug2" => LogLevel::Debug2MemoryAccesses,
"debug3" => LogLevel::Debug3ReadsFrom,
_ => return Err("invalid log level"),
})
}
}
#[cxx::bridge]
mod ffi {
/**** Types shared between Miri/Rust and Miri/C++ through cxx_bridge: ****/
/// Parameters that will be given to GenMC for setting up the model checker.
/// (The fields of this struct are visible to both Rust and C++)
/// The fields of this struct are visible to both Rust and C++.
/// Note that this struct is #[repr(C)], so the order of fields matters.
#[derive(Clone, Debug)]
struct GenmcParams {
/// Maximum number of executions explored in estimation mode.
pub estimation_max: u32,
pub print_random_schedule_seed: bool,
pub do_symmetry_reduction: bool,
// FIXME(GenMC): Add remaining parameters.
pub print_execution_graphs: ExecutiongraphPrinting,
/// Enabling this will set the memory model used by GenMC to "Sequential Consistency" (SC).
/// This will disable any weak memory effects, which reduces the number of program executions that will be explored.
pub disable_weak_memory_emulation: bool,
}
/// This is mostly equivalent to GenMC `VerbosityLevel`, but the debug log levels are always present (not conditionally compiled based on `ENABLE_GENMC_DEBUG`).
/// We add this intermediate type to prevent changes to the GenMC log-level from breaking the Miri
/// build, and to have a stable type for the C++-Rust interface, independent of `ENABLE_GENMC_DEBUG`.
#[derive(Debug)]
enum LogLevel {
/// Disable *all* logging (including error messages on a crash).
Quiet,
/// Log errors.
Error,
/// Log errors and warnings.
Warning,
/// Log errors, warnings and tips.
Tip,
/// Debug print considered revisits.
/// Downgraded to `Tip` if `GENMC_DEBUG` is not enabled.
Debug1Revisits,
/// Print the execution graph after every memory access.
/// Also includes the previous debug log level.
/// Downgraded to `Tip` if `GENMC_DEBUG` is not enabled.
Debug2MemoryAccesses,
/// Print reads-from values considered by GenMC.
/// Also includes the previous debug log level.
/// Downgraded to `Tip` if `GENMC_DEBUG` is not enabled.
Debug3ReadsFrom,
}
#[derive(Debug)]
/// Setting to control which execution graphs GenMC prints after every execution.
enum ExecutiongraphPrinting {
/// Print no graphs.
None,
/// Print graphs of all fully explored executions.
Explored,
/// Print graphs of all blocked executions.
Blocked,
/// Print graphs of all executions.
ExploredAndBlocked,
}
/// This type corresponds to `Option<SVal>` (or `std::optional<SVal>`), where `SVal` is the type that GenMC uses for storing values.
/// CXX doesn't support `std::optional` currently, so we need to use an extra `bool` to define whether this value is initialized or not.
#[derive(Debug, Clone, Copy)]
struct GenmcScalar {
value: u64,
is_init: bool,
}
#[must_use]
#[derive(Debug, Clone, Copy)]
enum ExecutionState {
Ok,
Blocked,
Finished,
}
#[must_use]
#[derive(Debug)]
struct SchedulingResult {
exec_state: ExecutionState,
next_thread: i32,
}
#[must_use]
#[derive(Debug)]
struct EstimationResult {
/// Expected number of total executions.
mean: f64,
/// Standard deviation of the total executions estimate.
sd: f64,
/// Number of explored executions during the estimation.
explored_execs: u64,
/// Number of encounteded blocked executions during the estimation.
blocked_execs: u64,
}
#[must_use]
#[derive(Debug)]
struct LoadResult {
/// If not null, contains the error encountered during the handling of the load.
error: UniquePtr<CxxString>,
/// Indicates whether a value was read or not.
has_value: bool,
/// The value that was read. Should not be used if `has_value` is `false`.
read_value: GenmcScalar,
}
#[must_use]
#[derive(Debug)]
struct StoreResult {
/// If not null, contains the error encountered during the handling of the store.
error: UniquePtr<CxxString>,
/// `true` if the write should also be reflected in Miri's memory representation.
is_coherence_order_maximal_write: bool,
}
#[must_use]
#[derive(Debug)]
struct ReadModifyWriteResult {
/// If there was an error, it will be stored in `error`, otherwise it is `None`.
error: UniquePtr<CxxString>,
/// The value that was read by the RMW operation as the left operand.
old_value: GenmcScalar,
/// The value that was produced by the RMW operation.
new_value: GenmcScalar,
/// `true` if the write should also be reflected in Miri's memory representation.
is_coherence_order_maximal_write: bool,
}
#[must_use]
#[derive(Debug)]
struct CompareExchangeResult {
/// If there was an error, it will be stored in `error`, otherwise it is `None`.
error: UniquePtr<CxxString>,
/// The value that was read by the compare-exchange.
old_value: GenmcScalar,
/// `true` if compare_exchange op was successful.
is_success: bool,
/// `true` if the write should also be reflected in Miri's memory representation.
is_coherence_order_maximal_write: bool,
}
/**** These are GenMC types that we have to copy-paste here since cxx does not support
"importing" externally defined C++ types. ****/
#[derive(Clone, Copy, Debug)]
enum SchedulePolicy {
LTR,
WF,
WFR,
Arbitrary,
}
#[derive(Debug)]
/// Corresponds to GenMC's type with the same name.
/// Should only be modified if changed by GenMC.
enum ActionKind {
/// Any Mir terminator that's atomic and has load semantics.
Load,
/// Anything that's not a `Load`.
NonLoad,
}
#[derive(Debug)]
/// Corresponds to GenMC's type with the same name.
/// Should only be modified if changed by GenMC.
enum MemOrdering {
NotAtomic = 0,
Relaxed = 1,
// We skip 2 in case we support consume.
Acquire = 3,
Release = 4,
AcquireRelease = 5,
SequentiallyConsistent = 6,
}
#[derive(Debug)]
enum RMWBinOp {
Xchg = 0,
Add = 1,
Sub = 2,
And = 3,
Nand = 4,
Or = 5,
Xor = 6,
Max = 7,
Min = 8,
UMax = 9,
UMin = 10,
}
// # Safety
//
// This block is unsafe to allow defining safe methods inside.
//
// `get_global_alloc_static_mask` is safe since it just returns a constant.
// All methods on `MiriGenmcShim` are safe by the correct usage of the two unsafe functions
// `set_log_level_raw` and `MiriGenmcShim::create_handle`.
// See the doc comment on those two functions for their safety requirements.
unsafe extern "C++" {
include!("MiriInterface.hpp");
type MiriGenMCShim;
/**** Types shared between Miri/Rust and Miri/C++: ****/
type MiriGenmcShim;
fn createGenmcHandle(config: &GenmcParams) -> UniquePtr<MiriGenMCShim>;
/**** Types shared between Miri/Rust and GenMC/C++:
(This tells cxx that the enums defined above are already defined on the C++ side;
it will emit assertions to ensure that the two definitions agree.) ****/
type ActionKind;
type MemOrdering;
type RMWBinOp;
type SchedulePolicy;
/// Set the log level for GenMC.
///
/// # Safety
///
/// This function is not thread safe, since it writes to the global, mutable, non-atomic `logLevel` variable.
/// Any GenMC function may read from `logLevel` unsynchronized.
/// The safest way to use this function is to set the log level exactly once before first calling `create_handle`.
/// Never calling this function is safe, GenMC will fall back to its default log level.
unsafe fn set_log_level_raw(log_level: LogLevel);
/// Create a new `MiriGenmcShim`, which wraps a `GenMCDriver`.
///
/// # Safety
///
/// This function is marked as unsafe since the `logLevel` global variable is non-atomic.
/// This function should not be called in an unsynchronized way with `set_log_level_raw`, since
/// this function and any methods on the returned `MiriGenmcShim` may read the `logLevel`,
/// causing a data race.
/// The safest way to use these functions is to call `set_log_level_raw` once, and only then
/// start creating handles.
/// There should not be any other (safe) way to create a `MiriGenmcShim`.
#[Self = "MiriGenmcShim"]
unsafe fn create_handle(
params: &GenmcParams,
estimation_mode: bool,
) -> UniquePtr<MiriGenmcShim>;
/// Get the bit mask that GenMC expects for global memory allocations.
fn get_global_alloc_static_mask() -> u64;
/// This function must be called at the start of any execution, before any events are reported to GenMC.
fn handle_execution_start(self: Pin<&mut MiriGenmcShim>);
/// This function must be called at the end of any execution, even if an error was found during the execution.
/// Returns `null`, or a string containing an error message if an error occured.
fn handle_execution_end(self: Pin<&mut MiriGenmcShim>) -> UniquePtr<CxxString>;
/***** Functions for handling events encountered during program execution. *****/
/**** Memory access handling ****/
fn handle_load(
self: Pin<&mut MiriGenmcShim>,
thread_id: i32,
address: u64,
size: u64,
memory_ordering: MemOrdering,
old_value: GenmcScalar,
) -> LoadResult;
fn handle_read_modify_write(
self: Pin<&mut MiriGenmcShim>,
thread_id: i32,
address: u64,
size: u64,
rmw_op: RMWBinOp,
ordering: MemOrdering,
rhs_value: GenmcScalar,
old_value: GenmcScalar,
) -> ReadModifyWriteResult;
fn handle_compare_exchange(
self: Pin<&mut MiriGenmcShim>,
thread_id: i32,
address: u64,
size: u64,
expected_value: GenmcScalar,
new_value: GenmcScalar,
old_value: GenmcScalar,
success_ordering: MemOrdering,
fail_load_ordering: MemOrdering,
can_fail_spuriously: bool,
) -> CompareExchangeResult;
fn handle_store(
self: Pin<&mut MiriGenmcShim>,
thread_id: i32,
address: u64,
size: u64,
value: GenmcScalar,
old_value: GenmcScalar,
memory_ordering: MemOrdering,
) -> StoreResult;
fn handle_fence(
self: Pin<&mut MiriGenmcShim>,
thread_id: i32,
memory_ordering: MemOrdering,
);
/**** Memory (de)allocation ****/
fn handle_malloc(
self: Pin<&mut MiriGenmcShim>,
thread_id: i32,
size: u64,
alignment: u64,
) -> u64;
fn handle_free(self: Pin<&mut MiriGenmcShim>, thread_id: i32, address: u64);
/**** Thread management ****/
fn handle_thread_create(self: Pin<&mut MiriGenmcShim>, thread_id: i32, parent_id: i32);
fn handle_thread_join(self: Pin<&mut MiriGenmcShim>, thread_id: i32, child_id: i32);
fn handle_thread_finish(self: Pin<&mut MiriGenmcShim>, thread_id: i32, ret_val: u64);
fn handle_thread_kill(self: Pin<&mut MiriGenmcShim>, thread_id: i32);
/***** Exploration related functionality *****/
/// Ask the GenMC scheduler for a new thread to schedule and
/// return whether the execution is finished, blocked, or can continue.
/// Updates the next instruction kind for the given thread id.
fn schedule_next(
self: Pin<&mut MiriGenmcShim>,
curr_thread_id: i32,
curr_thread_next_instr_kind: ActionKind,
) -> SchedulingResult;
/// Check whether there are more executions to explore.
/// If there are more executions, this method prepares for the next execution and returns `true`.
fn is_exploration_done(self: Pin<&mut MiriGenmcShim>) -> bool;
/**** Result querying functionality. ****/
// NOTE: We don't want to share the `VerificationResult` type with the Rust side, since it
// is very large, uses features that CXX.rs doesn't support and may change as GenMC changes.
// Instead, we only use the result on the C++ side, and only expose these getter function to
// the Rust side.
// Each `GenMCDriver` contains one `VerificationResult`, and each `MiriGenmcShim` contains on `GenMCDriver`.
// GenMC builds up the content of the `struct VerificationResult` over the course of an exploration,
// but it's safe to look at it at any point, since it is only accessible through exactly one `MiriGenmcShim`.
// All these functions for querying the result can be safely called repeatedly and at any time,
// though the results may be incomplete if called before `handle_execution_end`.
/// Get the number of blocked executions encountered by GenMC (cast into a fixed with integer)
fn get_blocked_execution_count(self: &MiriGenmcShim) -> u64;
/// Get the number of executions explored by GenMC (cast into a fixed with integer)
fn get_explored_execution_count(self: &MiriGenmcShim) -> u64;
/// Get all messages that GenMC produced (errors, warnings), combined into one string.
fn get_result_message(self: &MiriGenmcShim) -> UniquePtr<CxxString>;
/// If an error occurred, return a string describing the error, otherwise, return `nullptr`.
fn get_error_string(self: &MiriGenmcShim) -> UniquePtr<CxxString>;
/**** Printing functionality. ****/
/// Get the results of a run in estimation mode.
fn get_estimation_results(self: &MiriGenmcShim) -> EstimationResult;
}
}

50
src/tools/miri/genmc-sys/src_cpp/MiriInterface.cpp
View file

@ -1,50 +0,0 @@
#include "MiriInterface.hpp"
#include "genmc-sys/src/lib.rs.h"
auto MiriGenMCShim::createHandle(const GenmcParams &config)
-> std::unique_ptr<MiriGenMCShim>
{
auto conf = std::make_shared<Config>();
// Miri needs all threads to be replayed, even fully completed ones.
conf->replayCompletedThreads = true;
// We only support the RC11 memory model for Rust.
conf->model = ModelType::RC11;
conf->printRandomScheduleSeed = config.print_random_schedule_seed;
// FIXME(genmc): disable any options we don't support currently:
conf->ipr = false;
conf->disableBAM = true;
conf->instructionCaching = false;
ERROR_ON(config.do_symmetry_reduction, "Symmetry reduction is currently unsupported in GenMC mode.");
conf->symmetryReduction = config.do_symmetry_reduction;
// FIXME(genmc): Should there be a way to change this option from Miri?
conf->schedulePolicy = SchedulePolicy::WF;
// FIXME(genmc): implement estimation mode:
conf->estimate = false;
conf->estimationMax = 1000;
const auto mode = conf->estimate ? GenMCDriver::Mode(GenMCDriver::EstimationMode{})
: GenMCDriver::Mode(GenMCDriver::VerificationMode{});
// Running Miri-GenMC without race detection is not supported.
// Disabling this option also changes the behavior of the replay scheduler to only schedule at atomic operations, which is required with Miri.
// This happens because Miri can generate multiple GenMC events for a single MIR terminator. Without this option,
// the scheduler might incorrectly schedule an atomic MIR terminator because the first event it creates is a non-atomic (e.g., `StorageLive`).
conf->disableRaceDetection = false;
// Miri can already check for unfreed memory. Also, GenMC cannot distinguish between memory
// that is allowed to leak and memory that is not.
conf->warnUnfreedMemory = false;
// FIXME(genmc): check config:
// checkConfigOptions(*conf);
auto driver = std::make_unique<MiriGenMCShim>(std::move(conf), mode);
return driver;
}

44
src/tools/miri/genmc-sys/src_cpp/MiriInterface.hpp
View file

@ -1,44 +0,0 @@
#ifndef GENMC_MIRI_INTERFACE_HPP
#define GENMC_MIRI_INTERFACE_HPP
#include "rust/cxx.h"
#include "config.h"
#include "Config/Config.hpp"
#include "Verification/GenMCDriver.hpp"
#include <iostream>
/**** Types available to Miri ****/
// Config struct defined on the Rust side and translated to C++ by cxx.rs:
struct GenmcParams;
struct MiriGenMCShim : private GenMCDriver
{
public:
MiriGenMCShim(std::shared_ptr<const Config> conf, Mode mode /* = VerificationMode{} */)
: GenMCDriver(std::move(conf), nullptr, mode)
{
std::cerr << "C++: GenMC handle created!" << std::endl;
}
virtual ~MiriGenMCShim()
{
std::cerr << "C++: GenMC handle destroyed!" << std::endl;
}
static std::unique_ptr<MiriGenMCShim> createHandle(const GenmcParams &config);
};
/**** Functions available to Miri ****/
// NOTE: CXX doesn't support exposing static methods to Rust currently, so we expose this function instead.
static inline auto createGenmcHandle(const GenmcParams &config) -> std::unique_ptr<MiriGenMCShim>
{
return MiriGenMCShim::createHandle(config);
}
#endif /* GENMC_MIRI_INTERFACE_HPP */

2
src/tools/miri/rust-version
View file

@ -1 +1 @@
51ff895062ba60a7cba53f57af928c3fb7b0f2f4
3f1552a273e43e15f6ed240d00e1efdd6a53e65e

78
src/tools/miri/src/alloc_addresses/address_generator.rs Normal file
View file

@ -0,0 +1,78 @@
use std::ops::Range;
use rand::Rng;
use rustc_abi::{Align, Size};
use rustc_const_eval::interpret::{InterpResult, interp_ok};
use rustc_middle::{err_exhaust, throw_exhaust};
/// Shifts `addr` to make it aligned with `align` by rounding `addr` to the smallest multiple
/// of `align` that is larger or equal to `addr`
fn align_addr(addr: u64, align: u64) -> u64 {
match addr % align {
0 => addr,
rem => addr.strict_add(align) - rem,
}
}
/// This provides the logic to generate addresses for memory allocations in a given address range.
#[derive(Debug)]
pub struct AddressGenerator {
/// This is used as a memory address when a new pointer is casted to an integer. It
/// is always larger than any address that was previously made part of a block.
next_base_addr: u64,
/// This is the last address that can be allocated.
end: u64,
}
impl AddressGenerator {
pub fn new(addr_range: Range<u64>) -> Self {
Self { next_base_addr: addr_range.start, end: addr_range.end }
}
/// Get the remaining range where this `AddressGenerator` can still allocate addresses.
pub fn get_remaining(&self) -> Range<u64> {
self.next_base_addr..self.end
}
/// Generate a new address with the specified size and alignment, using the given Rng to add some randomness.
/// The returned allocation is guaranteed not to overlap with any address ranges given out by the generator before.
/// Returns an error if the allocation request cannot be fulfilled.
pub fn generate<'tcx, R: Rng>(
&mut self,
size: Size,
align: Align,
rng: &mut R,
) -> InterpResult<'tcx, u64> {
// Leave some space to the previous allocation, to give it some chance to be less aligned.
// We ensure that `(self.next_base_addr + slack) % 16` is uniformly distributed.
let slack = rng.random_range(0..16);
// From next_base_addr + slack, round up to adjust for alignment.
let base_addr =
self.next_base_addr.checked_add(slack).ok_or_else(|| err_exhaust!(AddressSpaceFull))?;
let base_addr = align_addr(base_addr, align.bytes());
// Remember next base address. If this allocation is zero-sized, leave a gap of at
// least 1 to avoid two allocations having the same base address. (The logic in
// `alloc_id_from_addr` assumes unique addresses, and different function/vtable pointers
// need to be distinguishable!)
self.next_base_addr = base_addr
.checked_add(size.bytes().max(1))
.ok_or_else(|| err_exhaust!(AddressSpaceFull))?;
// Even if `Size` didn't overflow, we might still have filled up the address space.
if self.next_base_addr > self.end {
throw_exhaust!(AddressSpaceFull);
}
interp_ok(base_addr)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_align_addr() {
assert_eq!(align_addr(37, 4), 40);
assert_eq!(align_addr(44, 4), 44);
}
}

84
src/tools/miri/src/alloc_addresses/mod.rs
View file

@ -1,15 +1,16 @@
//! This module is responsible for managing the absolute addresses that allocations are located at,
//! and for casting between pointers and integers based on those addresses.
mod address_generator;
mod reuse_pool;
use std::cell::RefCell;
use std::cmp::max;
use rand::Rng;
use rustc_abi::{Align, Size};
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
use rustc_middle::ty::TyCtxt;
pub use self::address_generator::AddressGenerator;
use self::reuse_pool::ReusePool;
use crate::concurrency::VClock;
use crate::*;
@ -33,6 +34,8 @@ pub struct GlobalStateInner {
/// sorted by address. We cannot use a `HashMap` since we can be given an address that is offset
/// from the base address, and we need to find the `AllocId` it belongs to. This is not the
/// *full* inverse of `base_addr`; dead allocations have been removed.
/// Note that in GenMC mode, dead allocations are *not* removed -- and also, addresses are never
/// reused. This lets us use the address as a cross-execution-stable identifier for an allocation.
int_to_ptr_map: Vec<(u64, AllocId)>,
/// The base address for each allocation. We cannot put that into
/// `AllocExtra` because function pointers also have a base address, and
@ -49,9 +52,8 @@ pub struct GlobalStateInner {
/// Whether an allocation has been exposed or not. This cannot be put
/// into `AllocExtra` for the same reason as `base_addr`.
exposed: FxHashSet<AllocId>,
/// This is used as a memory address when a new pointer is casted to an integer. It
/// is always larger than any address that was previously made part of a block.
next_base_addr: u64,
/// The generator for new addresses in a given range.
address_generator: AddressGenerator,
/// The provenance to use for int2ptr casts
provenance_mode: ProvenanceMode,
}
@ -64,7 +66,7 @@ impl VisitProvenance for GlobalStateInner {
prepared_alloc_bytes: _,
reuse: _,
exposed: _,
next_base_addr: _,
address_generator: _,
provenance_mode: _,
} = self;
// Though base_addr, int_to_ptr_map, and exposed contain AllocIds, we do not want to visit them.
@ -77,14 +79,14 @@ impl VisitProvenance for GlobalStateInner {
}
impl GlobalStateInner {
pub fn new(config: &MiriConfig, stack_addr: u64) -> Self {
pub fn new<'tcx>(config: &MiriConfig, stack_addr: u64, tcx: TyCtxt<'tcx>) -> Self {
GlobalStateInner {
int_to_ptr_map: Vec::default(),
base_addr: FxHashMap::default(),
prepared_alloc_bytes: FxHashMap::default(),
reuse: ReusePool::new(config),
exposed: FxHashSet::default(),
next_base_addr: stack_addr,
address_generator: AddressGenerator::new(stack_addr..tcx.target_usize_max()),
provenance_mode: config.provenance_mode,
}
}
@ -96,15 +98,6 @@ impl GlobalStateInner {
}
}
/// Shifts `addr` to make it aligned with `align` by rounding `addr` to the smallest multiple
/// of `align` that is larger or equal to `addr`
fn align_addr(addr: u64, align: u64) -> u64 {
match addr % align {
0 => addr,
rem => addr.strict_add(align) - rem,
}
}
impl<'tcx> EvalContextExtPriv<'tcx> for crate::MiriInterpCx<'tcx> {}
trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> {
fn addr_from_alloc_id_uncached(
@ -132,7 +125,8 @@ trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> {
// Miri's address assignment leaks state across thread boundaries, which is incompatible
// with GenMC execution. So we instead let GenMC assign addresses to allocations.
if let Some(genmc_ctx) = this.machine.data_race.as_genmc_ref() {
let addr = genmc_ctx.handle_alloc(&this.machine, info.size, info.align, memory_kind)?;
let addr =
genmc_ctx.handle_alloc(this, alloc_id, info.size, info.align, memory_kind)?;
return interp_ok(addr);
}
@ -189,39 +183,22 @@ trait EvalContextExtPriv<'tcx>: crate::MiriInterpCxExt<'tcx> {
this.active_thread(),
) {
if let Some(clock) = clock {
this.acquire_clock(&clock);
this.acquire_clock(&clock)?;
}
interp_ok(reuse_addr)
} else {
// We have to pick a fresh address.
// Leave some space to the previous allocation, to give it some chance to be less aligned.
// We ensure that `(global_state.next_base_addr + slack) % 16` is uniformly distributed.
let slack = rng.random_range(0..16);
// From next_base_addr + slack, round up to adjust for alignment.
let base_addr = global_state
.next_base_addr
.checked_add(slack)
.ok_or_else(|| err_exhaust!(AddressSpaceFull))?;
let base_addr = align_addr(base_addr, info.align.bytes());
let new_addr =
global_state.address_generator.generate(info.size, info.align, &mut rng)?;
// Remember next base address. If this allocation is zero-sized, leave a gap of at
// least 1 to avoid two allocations having the same base address. (The logic in
// `alloc_id_from_addr` assumes unique addresses, and different function/vtable pointers
// need to be distinguishable!)
global_state.next_base_addr = base_addr
.checked_add(max(info.size.bytes(), 1))
.ok_or_else(|| err_exhaust!(AddressSpaceFull))?;
// Even if `Size` didn't overflow, we might still have filled up the address space.
if global_state.next_base_addr > this.target_usize_max() {
throw_exhaust!(AddressSpaceFull);
}
// If we filled up more than half the address space, start aggressively reusing
// addresses to avoid running out.
if global_state.next_base_addr > u64::try_from(this.target_isize_max()).unwrap() {
let remaining_range = global_state.address_generator.get_remaining();
if remaining_range.start > remaining_range.end / 2 {
global_state.reuse.address_space_shortage();
}
interp_ok(base_addr)
interp_ok(new_addr)
}
}
}
@ -268,7 +245,10 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
// We only use this provenance if it has been exposed, or if the caller requested also non-exposed allocations
if !only_exposed_allocations || global_state.exposed.contains(&alloc_id) {
// This must still be live, since we remove allocations from `int_to_ptr_map` when they get freed.
debug_assert!(this.is_alloc_live(alloc_id));
// In GenMC mode, we keep all allocations, so this check doesn't apply there.
if this.machine.data_race.as_genmc_ref().is_none() {
debug_assert!(this.is_alloc_live(alloc_id));
}
Some(alloc_id)
} else {
None
@ -485,6 +465,13 @@ pub trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
impl<'tcx> MiriMachine<'tcx> {
pub fn free_alloc_id(&mut self, dead_id: AllocId, size: Size, align: Align, kind: MemoryKind) {
// In GenMC mode, we can't remove dead allocation info since such pointers can
// still be stored in atomics and we need this info to convert GenMC pointers to Miri pointers.
// `global_state.reuse` is also unused so we can just skip this entire function.
if self.data_race.as_genmc_ref().is_some() {
return;
}
let global_state = self.alloc_addresses.get_mut();
let rng = self.rng.get_mut();
@ -511,6 +498,8 @@ impl<'tcx> MiriMachine<'tcx> {
// Also remember this address for future reuse.
let thread = self.threads.active_thread();
global_state.reuse.add_addr(rng, addr, size, align, kind, thread, || {
// We already excluded GenMC above. We cannot use `self.release_clock` as
// `self.alloc_addresses` is borrowed.
if let Some(data_race) = self.data_race.as_vclocks_ref() {
data_race.release_clock(&self.threads, |clock| clock.clone())
} else {
@ -519,14 +508,3 @@ impl<'tcx> MiriMachine<'tcx> {
})
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_align_addr() {
assert_eq!(align_addr(37, 4), 40);
assert_eq!(align_addr(44, 4), 44);
}
}

36
src/tools/miri/src/bin/miri.rs
View file

@ -39,7 +39,7 @@ use std::sync::atomic::{AtomicI32, AtomicU32, Ordering};
use miri::{
BacktraceStyle, BorrowTrackerMethod, GenmcConfig, GenmcCtx, MiriConfig, MiriEntryFnType,
ProvenanceMode, RetagFields, TreeBorrowsParams, ValidationMode,
ProvenanceMode, RetagFields, TreeBorrowsParams, ValidationMode, run_genmc_mode,
};
use rustc_abi::ExternAbi;
use rustc_data_structures::sync;
@ -186,10 +186,18 @@ impl rustc_driver::Callbacks for MiriCompilerCalls {
optimizations is usually marginal at best.");
}
if let Some(_genmc_config) = &config.genmc_config {
let _genmc_ctx = Rc::new(GenmcCtx::new(&config));
todo!("GenMC mode not yet implemented");
// Run in GenMC mode if enabled.
if config.genmc_config.is_some() {
// This is the entry point used in GenMC mode.
// This closure will be called multiple times to explore the concurrent execution space of the program.
let eval_entry_once = |genmc_ctx: Rc<GenmcCtx>| {
miri::eval_entry(tcx, entry_def_id, entry_type, &config, Some(genmc_ctx))
};
let return_code = run_genmc_mode(&config, eval_entry_once, tcx).unwrap_or_else(|| {
tcx.dcx().abort_if_errors();
rustc_driver::EXIT_FAILURE
});
exit(return_code);
};
if let Some(many_seeds) = self.many_seeds.take() {
@ -737,19 +745,11 @@ fn main() {
many_seeds.map(|seeds| ManySeedsConfig { seeds, keep_going: many_seeds_keep_going });
// Validate settings for data race detection and GenMC mode.
if miri_config.genmc_config.is_some() {
if !miri_config.data_race_detector {
fatal_error!("Cannot disable data race detection in GenMC mode (currently)");
} else if !miri_config.weak_memory_emulation {
fatal_error!("Cannot disable weak memory emulation in GenMC mode");
}
if miri_config.borrow_tracker.is_some() {
eprintln!(
"warning: borrow tracking has been disabled, it is not (yet) supported in GenMC mode."
);
miri_config.borrow_tracker = None;
}
} else if miri_config.weak_memory_emulation && !miri_config.data_race_detector {
if let Err(err) = GenmcConfig::validate_genmc_mode_settings(&mut miri_config) {
fatal_error!("Invalid settings: {err}");
}
if miri_config.weak_memory_emulation && !miri_config.data_race_detector {
fatal_error!(
"Weak memory emulation cannot be enabled when the data race detector is disabled"
);

1
src/tools/miri/src/borrow_tracker/stacked_borrows/mod.rs
View file

@ -740,6 +740,7 @@ trait EvalContextPrivExt<'tcx, 'ecx>: crate::MiriInterpCxExt<'tcx> {
if let Some(access) = access {
assert_eq!(access, AccessKind::Write);
// Make sure the data race model also knows about this.
// FIXME(genmc): Ensure this is still done in GenMC mode. Check for other places where GenMC may need to be informed.
if let Some(data_race) = alloc_extra.data_race.as_vclocks_mut() {
data_race.write(
alloc_id,

2
src/tools/miri/src/borrow_tracker/tree_borrows/tree.rs
View file

@ -756,6 +756,8 @@ impl<'tcx> Tree {
// Don't check for protector if it is a Cell (see `unsafe_cell_deallocate` in `interior_mutability.rs`).
// Related to https://github.com/rust-lang/rust/issues/55005.
&& !perm.permission().is_cell()
// Only trigger UB if the accessed bit is set, i.e. if the protector is actually protecting this offset. See #4579.
&& perm.is_accessed()
{
Err(TransitionError::ProtectedDealloc)
} else {

23
src/tools/miri/src/clock.rs
View file

@ -46,14 +46,21 @@ impl Instant {
InstantKind::Virtual { nanoseconds: earlier },
) => {
let duration = nanoseconds.saturating_sub(earlier);
// `Duration` does not provide a nice constructor from a `u128` of nanoseconds,
// so we have to implement this ourselves.
// It is possible for second to overflow because u64::MAX < (u128::MAX / 1e9).
// It will be saturated to u64::MAX seconds if the value after division exceeds u64::MAX.
let seconds = u64::try_from(duration / 1_000_000_000).unwrap_or(u64::MAX);
// It is impossible for nanosecond to overflow because u32::MAX > 1e9.
let nanosecond = u32::try_from(duration.wrapping_rem(1_000_000_000)).unwrap();
Duration::new(seconds, nanosecond)
cfg_select! {
bootstrap => {
// `Duration` does not provide a nice constructor from a `u128` of nanoseconds,
// so we have to implement this ourselves.
// It is possible for second to overflow because u64::MAX < (u128::MAX / 1e9).
// It will be saturated to u64::MAX seconds if the value after division exceeds u64::MAX.
let seconds = u64::try_from(duration / 1_000_000_000).unwrap_or(u64::MAX);
// It is impossible for nanosecond to overflow because u32::MAX > 1e9.
let nanosecond = u32::try_from(duration.wrapping_rem(1_000_000_000)).unwrap();
Duration::new(seconds, nanosecond)
}
_ => {
Duration::from_nanos_u128(duration)
}
}
}
_ => panic!("all `Instant` must be of the same kind"),
}

209
src/tools/miri/src/concurrency/data_race.rs
View file

@ -56,6 +56,7 @@ use super::vector_clock::{VClock, VTimestamp, VectorIdx};
use super::weak_memory::EvalContextExt as _;
use crate::concurrency::GlobalDataRaceHandler;
use crate::diagnostics::RacingOp;
use crate::intrinsics::AtomicRmwOp;
use crate::*;
pub type AllocState = VClockAlloc;
@ -182,6 +183,9 @@ struct AtomicMemoryCellClocks {
/// contains the vector of timestamps that will
/// happen-before a thread if an acquire-load is
/// performed on the data.
///
/// With weak memory emulation, this is the clock of the most recent write. It is then only used
/// for release sequences, to integrate the most recent clock into the next one for RMWs.
sync_vector: VClock,
/// The size of accesses to this atomic location.
@ -275,7 +279,7 @@ struct MemoryCellClocks {
/// zero on each write operation.
read: VClock,
/// Atomic access, acquire, release sequence tracking clocks.
/// Atomic access tracking clocks.
/// For non-atomic memory this value is set to None.
/// For atomic memory, each byte carries this information.
atomic_ops: Option<Box<AtomicMemoryCellClocks>>,
@ -503,10 +507,11 @@ impl MemoryCellClocks {
thread_clocks: &mut ThreadClockSet,
index: VectorIdx,
access_size: Size,
sync_clock: Option<&VClock>,
) -> Result<(), DataRace> {
self.atomic_read_detect(thread_clocks, index, access_size)?;
if let Some(atomic) = self.atomic() {
thread_clocks.clock.join(&atomic.sync_vector);
if let Some(sync_clock) = sync_clock.or_else(|| self.atomic().map(|a| &a.sync_vector)) {
thread_clocks.clock.join(sync_clock);
}
Ok(())
}
@ -519,10 +524,11 @@ impl MemoryCellClocks {
thread_clocks: &mut ThreadClockSet,
index: VectorIdx,
access_size: Size,
sync_clock: Option<&VClock>,
) -> Result<(), DataRace> {
self.atomic_read_detect(thread_clocks, index, access_size)?;
if let Some(atomic) = self.atomic() {
thread_clocks.fence_acquire.join(&atomic.sync_vector);
if let Some(sync_clock) = sync_clock.or_else(|| self.atomic().map(|a| &a.sync_vector)) {
thread_clocks.fence_acquire.join(sync_clock);
}
Ok(())
}
@ -554,7 +560,8 @@ impl MemoryCellClocks {
// The handling of release sequences was changed in C++20 and so
// the code here is different to the paper since now all relaxed
// stores block release sequences. The exception for same-thread
// relaxed stores has been removed.
// relaxed stores has been removed. We always overwrite the `sync_vector`,
// meaning the previous release sequence is broken.
let atomic = self.atomic_mut_unwrap();
atomic.sync_vector.clone_from(&thread_clocks.fence_release);
Ok(())
@ -570,6 +577,8 @@ impl MemoryCellClocks {
) -> Result<(), DataRace> {
self.atomic_write_detect(thread_clocks, index, access_size)?;
let atomic = self.atomic_mut_unwrap();
// This *joining* of `sync_vector` implements release sequences: future
// reads of this location will acquire our clock *and* what was here before.
atomic.sync_vector.join(&thread_clocks.clock);
Ok(())
}
@ -584,6 +593,8 @@ impl MemoryCellClocks {
) -> Result<(), DataRace> {
self.atomic_write_detect(thread_clocks, index, access_size)?;
let atomic = self.atomic_mut_unwrap();
// This *joining* of `sync_vector` implements release sequences: future
// reads of this location will acquire our fence clock *and* what was here before.
atomic.sync_vector.join(&thread_clocks.fence_release);
Ok(())
}
@ -719,8 +730,7 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> {
// Only metadata on the location itself is used.
if let Some(genmc_ctx) = this.machine.data_race.as_genmc_ref() {
// FIXME(GenMC): Inform GenMC what a non-atomic read here would return, to support mixed atomics/non-atomics
let old_val = None;
let old_val = this.run_for_validation_ref(|this| this.read_scalar(place)).discard_err();
return genmc_ctx.atomic_load(
this,
place.ptr().addr(),
@ -731,8 +741,8 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> {
}
let scalar = this.allow_data_races_ref(move |this| this.read_scalar(place))?;
let buffered_scalar = this.buffered_atomic_read(place, atomic, scalar, || {
this.validate_atomic_load(place, atomic)
let buffered_scalar = this.buffered_atomic_read(place, atomic, scalar, |sync_clock| {
this.validate_atomic_load(place, atomic, sync_clock)
})?;
interp_ok(buffered_scalar.ok_or_else(|| err_ub!(InvalidUninitBytes(None)))?)
}
@ -751,11 +761,22 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> {
// The program didn't actually do a read, so suppress the memory access hooks.
// This is also a very special exception where we just ignore an error -- if this read
// was UB e.g. because the memory is uninitialized, we don't want to know!
let old_val = this.run_for_validation_mut(|this| this.read_scalar(dest)).discard_err();
let old_val = this.run_for_validation_ref(|this| this.read_scalar(dest)).discard_err();
// Inform GenMC about the atomic store.
if let Some(genmc_ctx) = this.machine.data_race.as_genmc_ref() {
// FIXME(GenMC): Inform GenMC what a non-atomic read here would return, to support mixed atomics/non-atomics
genmc_ctx.atomic_store(this, dest.ptr().addr(), dest.layout.size, val, atomic)?;
if genmc_ctx.atomic_store(
this,
dest.ptr().addr(),
dest.layout.size,
val,
old_val,
atomic,
)? {
// The store might be the latest store in coherence order (determined by GenMC).
// If it is, we need to update the value in Miri's memory:
this.allow_data_races_mut(|this| this.write_scalar(val, dest))?;
}
return interp_ok(());
}
this.allow_data_races_mut(move |this| this.write_scalar(val, dest))?;
@ -768,9 +789,8 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> {
&mut self,
place: &MPlaceTy<'tcx>,
rhs: &ImmTy<'tcx>,
op: mir::BinOp,
not: bool,
atomic: AtomicRwOrd,
atomic_op: AtomicRmwOp,
ord: AtomicRwOrd,
) -> InterpResult<'tcx, ImmTy<'tcx>> {
let this = self.eval_context_mut();
this.atomic_access_check(place, AtomicAccessType::Rmw)?;
@ -779,26 +799,42 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> {
// Inform GenMC about the atomic rmw operation.
if let Some(genmc_ctx) = this.machine.data_race.as_genmc_ref() {
// FIXME(GenMC): Inform GenMC what a non-atomic read here would return, to support mixed atomics/non-atomics
let (old_val, new_val) = genmc_ctx.atomic_rmw_op(
this,
place.ptr().addr(),
place.layout.size,
atomic,
(op, not),
atomic_op,
place.layout.backend_repr.is_signed(),
ord,
rhs.to_scalar(),
old.to_scalar(),
)?;
this.allow_data_races_mut(|this| this.write_scalar(new_val, place))?;
if let Some(new_val) = new_val {
this.allow_data_races_mut(|this| this.write_scalar(new_val, place))?;
}
return interp_ok(ImmTy::from_scalar(old_val, old.layout));
}
let val = this.binary_op(op, &old, rhs)?;
let val = if not { this.unary_op(mir::UnOp::Not, &val)? } else { val };
let val = match atomic_op {
AtomicRmwOp::MirOp { op, neg } => {
let val = this.binary_op(op, &old, rhs)?;
if neg { this.unary_op(mir::UnOp::Not, &val)? } else { val }
}
AtomicRmwOp::Max => {
let lt = this.binary_op(mir::BinOp::Lt, &old, rhs)?.to_scalar().to_bool()?;
if lt { rhs } else { &old }.clone()
}
AtomicRmwOp::Min => {
let lt = this.binary_op(mir::BinOp::Lt, &old, rhs)?.to_scalar().to_bool()?;
if lt { &old } else { rhs }.clone()
}
};
this.allow_data_races_mut(|this| this.write_immediate(*val, place))?;
this.validate_atomic_rmw(place, atomic)?;
this.validate_atomic_rmw(place, ord)?;
this.buffered_atomic_rmw(val.to_scalar(), place, atomic, old.to_scalar())?;
this.buffered_atomic_rmw(val.to_scalar(), place, ord, old.to_scalar())?;
interp_ok(old)
}
@ -818,14 +854,19 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> {
// Inform GenMC about the atomic atomic exchange.
if let Some(genmc_ctx) = this.machine.data_race.as_genmc_ref() {
// FIXME(GenMC): Inform GenMC what a non-atomic read here would return, to support mixed atomics/non-atomics
let (old_val, _is_success) = genmc_ctx.atomic_exchange(
let (old_val, new_val) = genmc_ctx.atomic_exchange(
this,
place.ptr().addr(),
place.layout.size,
new,
atomic,
old,
)?;
// The store might be the latest store in coherence order (determined by GenMC).
// If it is, we need to update the value in Miri's memory:
if let Some(new_val) = new_val {
this.allow_data_races_mut(|this| this.write_scalar(new_val, place))?;
}
return interp_ok(old_val);
}
@ -835,55 +876,6 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> {
interp_ok(old)
}
/// Perform an conditional atomic exchange with a memory place and a new
/// scalar value, the old value is returned.
fn atomic_min_max_scalar(
&mut self,
place: &MPlaceTy<'tcx>,
rhs: ImmTy<'tcx>,
min: bool,
atomic: AtomicRwOrd,
) -> InterpResult<'tcx, ImmTy<'tcx>> {
let this = self.eval_context_mut();
this.atomic_access_check(place, AtomicAccessType::Rmw)?;
let old = this.allow_data_races_mut(|this| this.read_immediate(place))?;
// Inform GenMC about the atomic min/max operation.
if let Some(genmc_ctx) = this.machine.data_race.as_genmc_ref() {
// FIXME(GenMC): Inform GenMC what a non-atomic read here would return, to support mixed atomics/non-atomics
let (old_val, new_val) = genmc_ctx.atomic_min_max_op(
this,
place.ptr().addr(),
place.layout.size,
atomic,
min,
old.layout.backend_repr.is_signed(),
rhs.to_scalar(),
)?;
this.allow_data_races_mut(|this| this.write_scalar(new_val, place))?;
return interp_ok(ImmTy::from_scalar(old_val, old.layout));
}
let lt = this.binary_op(mir::BinOp::Lt, &old, &rhs)?.to_scalar().to_bool()?;
#[rustfmt::skip] // rustfmt makes this unreadable
let new_val = if min {
if lt { &old } else { &rhs }
} else {
if lt { &rhs } else { &old }
};
this.allow_data_races_mut(|this| this.write_immediate(**new_val, place))?;
this.validate_atomic_rmw(place, atomic)?;
this.buffered_atomic_rmw(new_val.to_scalar(), place, atomic, old.to_scalar())?;
// Return the old value.
interp_ok(old)
}
/// Perform an atomic compare and exchange at a given memory location.
/// On success an atomic RMW operation is performed and on failure
/// only an atomic read occurs. If `can_fail_spuriously` is true,
@ -903,15 +895,12 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> {
let this = self.eval_context_mut();
this.atomic_access_check(place, AtomicAccessType::Rmw)?;
// Failure ordering cannot be stronger than success ordering, therefore first attempt
// to read with the failure ordering and if successful then try again with the success
// read ordering and write in the success case.
// Read as immediate for the sake of `binary_op()`
let old = this.allow_data_races_mut(|this| this.read_immediate(place))?;
// Inform GenMC about the atomic atomic compare exchange.
if let Some(genmc_ctx) = this.machine.data_race.as_genmc_ref() {
let (old, cmpxchg_success) = genmc_ctx.atomic_compare_exchange(
let (old_value, new_value, cmpxchg_success) = genmc_ctx.atomic_compare_exchange(
this,
place.ptr().addr(),
place.layout.size,
@ -920,11 +909,14 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> {
success,
fail,
can_fail_spuriously,
old.to_scalar(),
)?;
if cmpxchg_success {
this.allow_data_races_mut(|this| this.write_scalar(new, place))?;
// The store might be the latest store in coherence order (determined by GenMC).
// If it is, we need to update the value in Miri's memory:
if let Some(new_value) = new_value {
this.allow_data_races_mut(|this| this.write_scalar(new_value, place))?;
}
return interp_ok(Immediate::ScalarPair(old, Scalar::from_bool(cmpxchg_success)));
return interp_ok(Immediate::ScalarPair(old_value, Scalar::from_bool(cmpxchg_success)));
}
// `binary_op` will bail if either of them is not a scalar.
@ -948,7 +940,7 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> {
this.validate_atomic_rmw(place, success)?;
this.buffered_atomic_rmw(new, place, success, old.to_scalar())?;
} else {
this.validate_atomic_load(place, fail)?;
this.validate_atomic_load(place, fail, /* can use latest sync clock */ None)?;
// A failed compare exchange is equivalent to a load, reading from the latest store
// in the modification order.
// Since `old` is only a value and not the store element, we need to separately
@ -976,20 +968,36 @@ pub trait EvalContextExt<'tcx>: MiriInterpCxExt<'tcx> {
/// with this program point.
///
/// The closure will only be invoked if data race handling is on.
fn release_clock<R>(&self, callback: impl FnOnce(&VClock) -> R) -> Option<R> {
fn release_clock<R>(
&self,
callback: impl FnOnce(&VClock) -> R,
) -> InterpResult<'tcx, Option<R>> {
let this = self.eval_context_ref();
Some(
this.machine.data_race.as_vclocks_ref()?.release_clock(&this.machine.threads, callback),
)
interp_ok(match &this.machine.data_race {
GlobalDataRaceHandler::None => None,
GlobalDataRaceHandler::Genmc(_genmc_ctx) =>
throw_unsup_format!(
"this operation performs synchronization that is not supported in GenMC mode"
),
GlobalDataRaceHandler::Vclocks(data_race) =>
Some(data_race.release_clock(&this.machine.threads, callback)),
})
}
/// Acquire the given clock into the current thread, establishing synchronization with
/// the moment when that clock snapshot was taken via `release_clock`.
fn acquire_clock(&self, clock: &VClock) {
fn acquire_clock(&self, clock: &VClock) -> InterpResult<'tcx> {
let this = self.eval_context_ref();
if let Some(data_race) = this.machine.data_race.as_vclocks_ref() {
data_race.acquire_clock(clock, &this.machine.threads);
match &this.machine.data_race {
GlobalDataRaceHandler::None => {}
GlobalDataRaceHandler::Genmc(_genmc_ctx) =>
throw_unsup_format!(
"this operation performs synchronization that is not supported in GenMC mode"
),
GlobalDataRaceHandler::Vclocks(data_race) =>
data_race.acquire_clock(clock, &this.machine.threads),
}
interp_ok(())
}
}
@ -1174,6 +1182,18 @@ impl VClockAlloc {
}))?
}
/// Return the release/acquire synchronization clock for the given memory range.
pub(super) fn sync_clock(&self, access_range: AllocRange) -> VClock {
let alloc_ranges = self.alloc_ranges.borrow();
let mut clock = VClock::default();
for (_, mem_clocks) in alloc_ranges.iter(access_range.start, access_range.size) {
if let Some(atomic) = mem_clocks.atomic() {
clock.join(&atomic.sync_vector);
}
}
clock
}
/// Detect data-races for an unsynchronized read operation. It will not perform
/// data-race detection if `race_detecting()` is false, either due to no threads
/// being created or if it is temporarily disabled during a racy read or write
@ -1450,6 +1470,7 @@ trait EvalContextPrivExt<'tcx>: MiriInterpCxExt<'tcx> {
&self,
place: &MPlaceTy<'tcx>,
atomic: AtomicReadOrd,
sync_clock: Option<&VClock>,
) -> InterpResult<'tcx> {
let this = self.eval_context_ref();
this.validate_atomic_op(
@ -1458,9 +1479,9 @@ trait EvalContextPrivExt<'tcx>: MiriInterpCxExt<'tcx> {
AccessType::AtomicLoad,
move |memory, clocks, index, atomic| {
if atomic == AtomicReadOrd::Relaxed {
memory.load_relaxed(&mut *clocks, index, place.layout.size)
memory.load_relaxed(&mut *clocks, index, place.layout.size, sync_clock)
} else {
memory.load_acquire(&mut *clocks, index, place.layout.size)
memory.load_acquire(&mut *clocks, index, place.layout.size, sync_clock)
}
},
)
@ -1505,9 +1526,9 @@ trait EvalContextPrivExt<'tcx>: MiriInterpCxExt<'tcx> {
AccessType::AtomicRmw,
move |memory, clocks, index, _| {
if acquire {
memory.load_acquire(clocks, index, place.layout.size)?;
memory.load_acquire(clocks, index, place.layout.size, None)?;
} else {
memory.load_relaxed(clocks, index, place.layout.size)?;
memory.load_relaxed(clocks, index, place.layout.size, None)?;
}
if release {
memory.rmw_release(clocks, index, place.layout.size)

92
src/tools/miri/src/concurrency/genmc/config.rs
View file

@ -1,13 +1,24 @@
use genmc_sys::LogLevel;
use super::GenmcParams;
use crate::{IsolatedOp, MiriConfig, RejectOpWith};
/// Configuration for GenMC mode.
/// The `params` field is shared with the C++ side.
/// The remaining options are kept on the Rust side.
#[derive(Debug, Default, Clone)]
pub struct GenmcConfig {
/// Parameters sent to the C++ side to create a new handle to the GenMC model checker.
pub(super) params: GenmcParams,
do_estimation: bool,
// FIXME(GenMC): add remaining options.
pub(super) do_estimation: bool,
/// Print the output message that GenMC generates when an error occurs.
/// This error message is currently hard to use, since there is no clear mapping between the events that GenMC sees and the Rust code location where this event was produced.
pub(super) print_genmc_output: bool,
/// The log level for GenMC.
pub(super) log_level: LogLevel,
/// Enable more verbose output, such as number of executions estimate
/// and time to completion of verification step.
pub(super) verbose_output: bool,
}
impl GenmcConfig {
@ -29,7 +40,80 @@ impl GenmcConfig {
if trimmed_arg.is_empty() {
return Ok(()); // this corresponds to "-Zmiri-genmc"
}
// FIXME(GenMC): implement remaining parameters.
todo!();
let genmc_config = genmc_config.as_mut().unwrap();
let Some(trimmed_arg) = trimmed_arg.strip_prefix("-") else {
return Err(format!("Invalid GenMC argument \"-Zmiri-genmc{trimmed_arg}\""));
};
if let Some(log_level) = trimmed_arg.strip_prefix("log=") {
genmc_config.log_level = log_level.parse()?;
} else if let Some(trimmed_arg) = trimmed_arg.strip_prefix("print-exec-graphs") {
use genmc_sys::ExecutiongraphPrinting;
genmc_config.params.print_execution_graphs = match trimmed_arg {
"=none" => ExecutiongraphPrinting::None,
// Make GenMC print explored executions.
"" | "=explored" => ExecutiongraphPrinting::Explored,
// Make GenMC print blocked executions.
"=blocked" => ExecutiongraphPrinting::Blocked,
// Make GenMC print all executions.
"=all" => ExecutiongraphPrinting::ExploredAndBlocked,
_ =>
return Err(format!(
"Invalid suffix to GenMC argument '-Zmiri-genmc-print-exec-graphs', expected '', '=none', '=explored', '=blocked' or '=all'"
)),
}
} else if trimmed_arg == "estimate" {
// FIXME(genmc): should this be on by default (like for GenMC)?
// Enable estimating the execution space and require time before running the actual verification.
genmc_config.do_estimation = true;
} else if let Some(estimation_max_str) = trimmed_arg.strip_prefix("estimation-max=") {
// Set the maximum number of executions to explore during estimation.
genmc_config.params.estimation_max = estimation_max_str.parse().ok().filter(|estimation_max| *estimation_max > 0).ok_or_else(|| {
format!(
"'-Zmiri-genmc-estimation-max=...' expects a positive integer argument, but got '{estimation_max_str}'"
)
})?;
} else if trimmed_arg == "print-genmc-output" {
genmc_config.print_genmc_output = true;
} else if trimmed_arg == "verbose" {
genmc_config.verbose_output = true;
} else {
return Err(format!("Invalid GenMC argument: \"-Zmiri-genmc-{trimmed_arg}\""));
}
Ok(())
}
/// Validate settings for GenMC mode (NOP if GenMC mode disabled).
///
/// Unsupported configurations return an error.
/// Adjusts Miri settings where required, printing a warnings if the change might be unexpected for the user.
pub fn validate_genmc_mode_settings(miri_config: &mut MiriConfig) -> Result<(), &'static str> {
let Some(genmc_config) = miri_config.genmc_config.as_mut() else {
return Ok(());
};
// Check for disallowed configurations.
if !miri_config.data_race_detector {
return Err("Cannot disable data race detection in GenMC mode");
} else if !miri_config.native_lib.is_empty() {
return Err("native-lib not supported in GenMC mode.");
} else if miri_config.isolated_op != IsolatedOp::Reject(RejectOpWith::Abort) {
return Err("Cannot disable isolation in GenMC mode");
}
// Adjust settings where needed.
if !miri_config.weak_memory_emulation {
genmc_config.params.disable_weak_memory_emulation = true;
}
if miri_config.borrow_tracker.is_some() {
eprintln!(
"warning: borrow tracking has been disabled, it is not (yet) supported in GenMC mode."
);
miri_config.borrow_tracker = None;
}
// We enable fixed scheduling so Miri doesn't randomly yield before a terminator, which anyway
// would be a NOP in GenMC mode.
miri_config.fixed_scheduling = true;
Ok(())
}
}

129
src/tools/miri/src/concurrency/genmc/dummy.rs
View file

@ -1,50 +1,46 @@
#![allow(unused)]
use rustc_abi::{Align, Size};
use rustc_const_eval::interpret::{InterpCx, InterpResult};
use rustc_middle::mir;
use rustc_const_eval::interpret::{AllocId, InterpCx, InterpResult};
pub use self::run::run_genmc_mode;
use crate::intrinsics::AtomicRmwOp;
use crate::{
AtomicFenceOrd, AtomicReadOrd, AtomicRwOrd, AtomicWriteOrd, MemoryKind, MiriConfig,
MiriMachine, Scalar, ThreadId, ThreadManager, VisitProvenance, VisitWith,
AtomicFenceOrd, AtomicReadOrd, AtomicRwOrd, AtomicWriteOrd, MemoryKind, MiriMachine, Scalar,
ThreadId, ThreadManager, VisitProvenance, VisitWith,
};
#[derive(Clone, Copy, Debug)]
pub enum ExitType {
MainThreadFinish,
ExitCalled,
}
#[derive(Debug)]
pub struct GenmcCtx {}
#[derive(Debug, Default, Clone)]
pub struct GenmcConfig {}
mod run {
use std::rc::Rc;
use rustc_middle::ty::TyCtxt;
use crate::{GenmcCtx, MiriConfig};
pub fn run_genmc_mode<'tcx>(
_config: &MiriConfig,
_eval_entry: impl Fn(Rc<GenmcCtx>) -> Option<i32>,
_tcx: TyCtxt<'tcx>,
) -> Option<i32> {
unreachable!();
}
}
impl GenmcCtx {
pub fn new(_miri_config: &MiriConfig) -> Self {
unreachable!()
}
pub fn get_stuck_execution_count(&self) -> usize {
unreachable!()
}
pub fn print_genmc_graph(&self) {
unreachable!()
}
pub fn is_exploration_done(&self) -> bool {
unreachable!()
}
// We don't provide the `new` function in the dummy module.
/**** Memory access handling ****/
pub(crate) fn handle_execution_start(&self) {
unreachable!()
}
pub(crate) fn handle_execution_end<'tcx>(
&self,
_ecx: &InterpCx<'tcx, MiriMachine<'tcx>>,
) -> Result<(), String> {
unreachable!()
}
pub(super) fn set_ongoing_action_data_race_free(&self, _enable: bool) {
unreachable!()
}
@ -67,8 +63,9 @@ impl GenmcCtx {
_address: Size,
_size: Size,
_value: Scalar,
_old_value: Option<Scalar>,
_ordering: AtomicWriteOrd,
) -> InterpResult<'tcx, ()> {
) -> InterpResult<'tcx, bool> {
unreachable!()
}
@ -76,7 +73,7 @@ impl GenmcCtx {
&self,
_machine: &MiriMachine<'tcx>,
_ordering: AtomicFenceOrd,
) -> InterpResult<'tcx, ()> {
) -> InterpResult<'tcx> {
unreachable!()
}
@ -85,23 +82,12 @@ impl GenmcCtx {
_ecx: &InterpCx<'tcx, MiriMachine<'tcx>>,
_address: Size,
_size: Size,
_atomic_op: AtomicRmwOp,
_is_signed: bool,
_ordering: AtomicRwOrd,
(rmw_op, not): (mir::BinOp, bool),
_rhs_scalar: Scalar,
) -> InterpResult<'tcx, (Scalar, Scalar)> {
unreachable!()
}
pub(crate) fn atomic_min_max_op<'tcx>(
&self,
ecx: &InterpCx<'tcx, MiriMachine<'tcx>>,
address: Size,
size: Size,
ordering: AtomicRwOrd,
min: bool,
is_signed: bool,
rhs_scalar: Scalar,
) -> InterpResult<'tcx, (Scalar, Scalar)> {
_old_value: Scalar,
) -> InterpResult<'tcx, (Scalar, Option<Scalar>)> {
unreachable!()
}
@ -112,7 +98,8 @@ impl GenmcCtx {
_size: Size,
_rhs_scalar: Scalar,
_ordering: AtomicRwOrd,
) -> InterpResult<'tcx, (Scalar, bool)> {
_old_value: Scalar,
) -> InterpResult<'tcx, (Scalar, Option<Scalar>)> {
unreachable!()
}
@ -126,7 +113,8 @@ impl GenmcCtx {
_success: AtomicRwOrd,
_fail: AtomicReadOrd,
_can_fail_spuriously: bool,
) -> InterpResult<'tcx, (Scalar, bool)> {
_old_value: Scalar,
) -> InterpResult<'tcx, (Scalar, Option<Scalar>, bool)> {
unreachable!()
}
@ -135,7 +123,7 @@ impl GenmcCtx {
_machine: &MiriMachine<'tcx>,
_address: Size,
_size: Size,
) -> InterpResult<'tcx, ()> {
) -> InterpResult<'tcx> {
unreachable!()
}
@ -144,7 +132,7 @@ impl GenmcCtx {
_machine: &MiriMachine<'tcx>,
_address: Size,
_size: Size,
) -> InterpResult<'tcx, ()> {
) -> InterpResult<'tcx> {
unreachable!()
}
@ -152,7 +140,8 @@ impl GenmcCtx {
pub(crate) fn handle_alloc<'tcx>(
&self,
_machine: &MiriMachine<'tcx>,
_ecx: &InterpCx<'tcx, MiriMachine<'tcx>>,
_alloc_id: AllocId,
_size: Size,
_alignment: Align,
_memory_kind: MemoryKind,
@ -163,11 +152,10 @@ impl GenmcCtx {
pub(crate) fn handle_dealloc<'tcx>(
&self,
_machine: &MiriMachine<'tcx>,
_alloc_id: AllocId,
_address: Size,
_size: Size,
_align: Align,
_kind: MemoryKind,
) -> InterpResult<'tcx, ()> {
) -> InterpResult<'tcx> {
unreachable!()
}
@ -176,8 +164,10 @@ impl GenmcCtx {
pub(crate) fn handle_thread_create<'tcx>(
&self,
_threads: &ThreadManager<'tcx>,
_start_routine: crate::Pointer,
_func_arg: &crate::ImmTy<'tcx>,
_new_thread_id: ThreadId,
) -> InterpResult<'tcx, ()> {
) -> InterpResult<'tcx> {
unreachable!()
}
@ -185,24 +175,26 @@ impl GenmcCtx {
&self,
_active_thread_id: ThreadId,
_child_thread_id: ThreadId,
) -> InterpResult<'tcx, ()> {
) -> InterpResult<'tcx> {
unreachable!()
}
pub(crate) fn handle_thread_stack_empty(&self, _thread_id: ThreadId) {
pub(crate) fn handle_thread_finish<'tcx>(&self, _threads: &ThreadManager<'tcx>) {
unreachable!()
}
pub(crate) fn handle_thread_finish<'tcx>(
pub(crate) fn handle_exit<'tcx>(
&self,
_threads: &ThreadManager<'tcx>,
) -> InterpResult<'tcx, ()> {
_thread: ThreadId,
_exit_code: i32,
_exit_type: ExitType,
) -> InterpResult<'tcx> {
unreachable!()
}
/**** Scheduling functionality ****/
pub(crate) fn schedule_thread<'tcx>(
pub fn schedule_thread<'tcx>(
&self,
_ecx: &InterpCx<'tcx, MiriMachine<'tcx>>,
) -> InterpResult<'tcx, ThreadId> {
@ -211,6 +203,7 @@ impl GenmcCtx {
/**** Blocking instructions ****/
#[allow(unused)]
pub(crate) fn handle_verifier_assume<'tcx>(
&self,
_machine: &MiriMachine<'tcx>,
@ -229,7 +222,7 @@ impl VisitProvenance for GenmcCtx {
impl GenmcConfig {
pub fn parse_arg(
_genmc_config: &mut Option<GenmcConfig>,
trimmed_arg: &str,
_trimmed_arg: &str,
) -> Result<(), String> {
if cfg!(feature = "genmc") {
Err(format!("GenMC is disabled in this build of Miri"))
@ -238,7 +231,9 @@ impl GenmcConfig {
}
}
pub fn should_print_graph(&self, _rep: usize) -> bool {
unreachable!()
pub fn validate_genmc_mode_settings(
_miri_config: &mut crate::MiriConfig,
) -> Result<(), &'static str> {
Ok(())
}
}

118
src/tools/miri/src/concurrency/genmc/global_allocations.rs Normal file
View file

@ -0,0 +1,118 @@
use std::collections::hash_map::Entry;
use std::sync::RwLock;
use genmc_sys::{GENMC_GLOBAL_ADDRESSES_MASK, get_global_alloc_static_mask};
use rand::SeedableRng;
use rand::rngs::StdRng;
use rustc_const_eval::interpret::{AllocId, AllocInfo, InterpResult, interp_ok};
use rustc_data_structures::fx::FxHashMap;
use tracing::debug;
use crate::alloc_addresses::AddressGenerator;
#[derive(Debug)]
struct GlobalStateInner {
/// The base address for each *global* allocation.
base_addr: FxHashMap<AllocId, u64>,
/// We use the same address generator that Miri uses in normal operation.
address_generator: AddressGenerator,
/// The address generator needs an Rng to randomize the offsets between allocations.
/// We don't use the `MiriMachine` Rng since this is global, cross-machine state.
rng: StdRng,
}
/// Allocator for global memory in GenMC mode.
/// Miri doesn't discover all global allocations statically like LLI does for GenMC.
/// The existing global memory allocator in GenMC doesn't support this, so we take over these allocations.
/// Global allocations need to be in a specific address range, with the lower limit given by the `GENMC_GLOBAL_ADDRESSES_MASK` constant.
///
/// Every global allocation must have the same addresses across all executions of a single program.
/// Therefore there is only 1 global allocator, and it syncs new globals across executions, even if they are explored in parallel.
#[derive(Debug)]
pub struct GlobalAllocationHandler(RwLock<GlobalStateInner>);
impl GlobalAllocationHandler {
/// Create a new global address generator with a given max address `last_addr`
/// (corresponding to the highest address available on the target platform, unless another limit exists).
/// No addresses higher than this will be allocated.
/// Will panic if the given address limit is too small to allocate any addresses.
pub fn new(last_addr: u64) -> GlobalAllocationHandler {
assert_eq!(GENMC_GLOBAL_ADDRESSES_MASK, get_global_alloc_static_mask());
assert_ne!(GENMC_GLOBAL_ADDRESSES_MASK, 0);
// FIXME(genmc): Remove if non-64bit targets are supported.
assert!(
GENMC_GLOBAL_ADDRESSES_MASK < last_addr,
"only 64bit platforms are currently supported (highest address {last_addr:#x} <= minimum global address {GENMC_GLOBAL_ADDRESSES_MASK:#x})."
);
Self(RwLock::new(GlobalStateInner {
base_addr: FxHashMap::default(),
address_generator: AddressGenerator::new(GENMC_GLOBAL_ADDRESSES_MASK..last_addr),
// FIXME(genmc): We could provide a way to changes this seed, to allow for different global addresses.
rng: StdRng::seed_from_u64(0),
}))
}
}
impl GlobalStateInner {
fn global_allocate_addr<'tcx>(
&mut self,
alloc_id: AllocId,
info: AllocInfo,
) -> InterpResult<'tcx, u64> {
let entry = match self.base_addr.entry(alloc_id) {
Entry::Occupied(occupied_entry) => {
// Looks like some other thread allocated this for us
// between when we released the read lock and aquired the write lock,
// so we just return that value.
return interp_ok(*occupied_entry.get());
}
Entry::Vacant(vacant_entry) => vacant_entry,
};
// This allocation does not have a base address yet, pick or reuse one.
// We are not in native lib mode (incompatible with GenMC mode), so we control the addresses ourselves.
let new_addr = self.address_generator.generate(info.size, info.align, &mut self.rng)?;
// Cache the address for future use.
entry.insert(new_addr);
interp_ok(new_addr)
}
}
impl<'tcx> EvalContextExt<'tcx> for crate::MiriInterpCx<'tcx> {}
pub(super) trait EvalContextExt<'tcx>: crate::MiriInterpCxExt<'tcx> {
/// Allocate a new address for the given alloc id, or return the cached address.
/// Each alloc id is assigned one unique allocation which will not change if this function is called again with the same alloc id.
fn get_global_allocation_address(
&self,
global_allocation_handler: &GlobalAllocationHandler,
alloc_id: AllocId,
) -> InterpResult<'tcx, u64> {
let this = self.eval_context_ref();
let info = this.get_alloc_info(alloc_id);
let global_state = global_allocation_handler.0.read().unwrap();
if let Some(base_addr) = global_state.base_addr.get(&alloc_id) {
debug!(
"GenMC: address for global with alloc id {alloc_id:?} was cached: {base_addr} == {base_addr:#x}"
);
return interp_ok(*base_addr);
}
// We need to upgrade to a write lock. `std::sync::RwLock` doesn't support this, so we drop the guard and lock again
// Note that another thread might allocate the address while the `RwLock` is unlocked, but we handle this case in the allocation function.
drop(global_state);
let mut global_state = global_allocation_handler.0.write().unwrap();
// With the write lock, we can safely allocate an address only once per `alloc_id`.
let new_addr = global_state.global_allocate_addr(alloc_id, info)?;
debug!("GenMC: global with alloc id {alloc_id:?} got address: {new_addr} == {new_addr:#x}");
assert_eq!(
GENMC_GLOBAL_ADDRESSES_MASK,
new_addr & GENMC_GLOBAL_ADDRESSES_MASK,
"Global address allocated outside global address space."
);
interp_ok(new_addr)
}
}

Some files were not shown because too many files have changed in this diff Show more