From 9a7a5193706319b17c260da433f27c2c2c8f5b07 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Thu, 26 Dec 2024 07:44:54 +0000 Subject: [PATCH] Streamline the way that test iteration count is determined Currently, tests use a handful of constants to determine how many iterations to perform: `NTESTS`, `AROUND`, and `MAX_CHECK_POINTS`. This configuration is not very straightforward to adjust and needs to be repeated everywhere it is used. Replace this with new functions in the `run_cfg` module that determine iteration counts in a more reusable and documented way. This only updates `edge_cases` and `domain_logspace`, `random` is refactored in a later commit. --- .../libm-test/src/gen/domain_logspace.rs | 31 +-- .../crates/libm-test/src/gen/edge_cases.rs | 54 +++--- .../libm/crates/libm-test/src/gen/random.rs | 1 + .../libm/crates/libm-test/src/lib.rs | 2 +- .../libm/crates/libm-test/src/run_cfg.rs | 177 +++++++++++++++++- 5 files changed, 208 insertions(+), 57 deletions(-) diff --git a/library/compiler-builtins/libm/crates/libm-test/src/gen/domain_logspace.rs b/library/compiler-builtins/libm/crates/libm-test/src/gen/domain_logspace.rs index 5e37170fa1a4..3d8a3e7fe733 100644 --- a/library/compiler-builtins/libm/crates/libm-test/src/gen/domain_logspace.rs +++ b/library/compiler-builtins/libm/crates/libm-test/src/gen/domain_logspace.rs @@ -6,41 +6,26 @@ use libm::support::{IntTy, MinInt}; use crate::domain::HasDomain; use crate::op::OpITy; +use crate::run_cfg::{GeneratorKind, iteration_count}; use crate::{CheckCtx, MathOp, logspace}; -/// Number of tests to run. -// FIXME(ntests): replace this with a more logical algorithm -const NTESTS: usize = { - if cfg!(optimizations_enabled) { - if crate::emulated() - || !cfg!(target_pointer_width = "64") - || cfg!(all(target_arch = "x86_64", target_vendor = "apple")) - { - // Tests are pretty slow on non-64-bit targets, x86 MacOS, and targets that run - // in QEMU. - 100_000 - } else { - 5_000_000 - } - } else { - // Without optimizations just run a quick check - 800 - } -}; - /// Create a range of logarithmically spaced inputs within a function's domain. /// /// This allows us to get reasonably thorough coverage without wasting time on values that are /// NaN or out of range. Random tests will still cover values that are excluded here. -pub fn get_test_cases(_ctx: &CheckCtx) -> impl Iterator +pub fn get_test_cases(ctx: &CheckCtx) -> impl Iterator where Op: MathOp + HasDomain, - IntTy: TryFrom, + IntTy: TryFrom, RangeInclusive>: Iterator, { let domain = Op::DOMAIN; + let ntests = iteration_count(ctx, GeneratorKind::Domain, 0); + + // We generate logspaced inputs within a specific range, excluding values that are out of + // range in order to make iterations useful (random tests still cover the full range). let start = domain.range_start(); let end = domain.range_end(); - let steps = OpITy::::try_from(NTESTS).unwrap_or(OpITy::::MAX); + let steps = OpITy::::try_from(ntests).unwrap_or(OpITy::::MAX); logspace(start, end, steps).map(|v| (v,)) } diff --git a/library/compiler-builtins/libm/crates/libm-test/src/gen/edge_cases.rs b/library/compiler-builtins/libm/crates/libm-test/src/gen/edge_cases.rs index 3387f6c485ae..1f27c1467dcd 100644 --- a/library/compiler-builtins/libm/crates/libm-test/src/gen/edge_cases.rs +++ b/library/compiler-builtins/libm/crates/libm-test/src/gen/edge_cases.rs @@ -3,18 +3,11 @@ use libm::support::Float; use crate::domain::HasDomain; +use crate::run_cfg::{check_near_count, check_point_count}; use crate::{CheckCtx, FloatExt, MathOp}; -/// Number of values near an interesting point to check. -// FIXME(ntests): replace this with a more logical algorithm -const AROUND: usize = 100; - -/// Functions have infinite asymptotes, limit how many we check. -// FIXME(ntests): replace this with a more logical algorithm -const MAX_CHECK_POINTS: usize = 10; - /// Create a list of values around interesting points (infinities, zeroes, NaNs). -pub fn get_test_cases(_ctx: &CheckCtx) -> impl Iterator +pub fn get_test_cases(ctx: &CheckCtx) -> impl Iterator where Op: MathOp + HasDomain, F: Float, @@ -25,23 +18,26 @@ where let domain_start = domain.range_start(); let domain_end = domain.range_end(); + let check_points = check_point_count(ctx); + let near_points = check_near_count(ctx); + // Check near some notable constants - count_up(F::ONE, values); - count_up(F::ZERO, values); - count_up(F::NEG_ONE, values); - count_down(F::ONE, values); - count_down(F::ZERO, values); - count_down(F::NEG_ONE, values); + count_up(F::ONE, near_points, values); + count_up(F::ZERO, near_points, values); + count_up(F::NEG_ONE, near_points, values); + count_down(F::ONE, near_points, values); + count_down(F::ZERO, near_points, values); + count_down(F::NEG_ONE, near_points, values); values.push(F::NEG_ZERO); // Check values near the extremes - count_up(F::NEG_INFINITY, values); - count_down(F::INFINITY, values); - count_down(domain_end, values); - count_up(domain_start, values); - count_down(domain_start, values); - count_up(domain_end, values); - count_down(domain_end, values); + count_up(F::NEG_INFINITY, near_points, values); + count_down(F::INFINITY, near_points, values); + count_down(domain_end, near_points, values); + count_up(domain_start, near_points, values); + count_down(domain_start, near_points, values); + count_up(domain_end, near_points, values); + count_down(domain_end, near_points, values); // Check some special values that aren't included in the above ranges values.push(F::NAN); @@ -50,9 +46,9 @@ where // Check around asymptotes if let Some(f) = domain.check_points { let iter = f(); - for x in iter.take(MAX_CHECK_POINTS) { - count_up(x, values); - count_down(x, values); + for x in iter.take(check_points) { + count_up(x, near_points, values); + count_down(x, near_points, values); } } @@ -65,11 +61,11 @@ where /// Add `AROUND` values starting at and including `x` and counting up. Uses the smallest possible /// increments (1 ULP). -fn count_up(mut x: F, values: &mut Vec) { +fn count_up(mut x: F, points: u64, values: &mut Vec) { assert!(!x.is_nan()); let mut count = 0; - while x < F::INFINITY && count < AROUND { + while x < F::INFINITY && count < points { values.push(x); x = x.next_up(); count += 1; @@ -78,11 +74,11 @@ fn count_up(mut x: F, values: &mut Vec) { /// Add `AROUND` values starting at and including `x` and counting down. Uses the smallest possible /// increments (1 ULP). -fn count_down(mut x: F, values: &mut Vec) { +fn count_down(mut x: F, points: u64, values: &mut Vec) { assert!(!x.is_nan()); let mut count = 0; - while x > F::NEG_INFINITY && count < AROUND { + while x > F::NEG_INFINITY && count < points { values.push(x); x = x.next_down(); count += 1; diff --git a/library/compiler-builtins/libm/crates/libm-test/src/gen/random.rs b/library/compiler-builtins/libm/crates/libm-test/src/gen/random.rs index 4f75da07b898..a30a3674e6a4 100644 --- a/library/compiler-builtins/libm/crates/libm-test/src/gen/random.rs +++ b/library/compiler-builtins/libm/crates/libm-test/src/gen/random.rs @@ -12,6 +12,7 @@ use crate::{BaseName, CheckCtx, GenerateInput}; const SEED: [u8; 32] = *b"3.141592653589793238462643383279"; /// Number of tests to run. +// FIXME(ntests): clean this up when possible const NTESTS: usize = { if cfg!(optimizations_enabled) { if crate::emulated() diff --git a/library/compiler-builtins/libm/crates/libm-test/src/lib.rs b/library/compiler-builtins/libm/crates/libm-test/src/lib.rs index c1aec0230303..80ec23736157 100644 --- a/library/compiler-builtins/libm/crates/libm-test/src/lib.rs +++ b/library/compiler-builtins/libm/crates/libm-test/src/lib.rs @@ -25,7 +25,7 @@ pub use libm::support::{Float, Int, IntTy, MinInt}; pub use num::{FloatExt, logspace}; pub use op::{BaseName, FloatTy, Identifier, MathOp, OpCFn, OpFTy, OpRustFn, OpRustRet, Ty}; pub use precision::{MaybeOverride, SpecialCase, default_ulp}; -pub use run_cfg::{CheckBasis, CheckCtx}; +pub use run_cfg::{CheckBasis, CheckCtx, EXTENSIVE_ENV, GeneratorKind}; pub use test_traits::{CheckOutput, GenerateInput, Hex, TupleCall}; /// Result type for tests is usually from `anyhow`. Most times there is no success value to diff --git a/library/compiler-builtins/libm/crates/libm-test/src/run_cfg.rs b/library/compiler-builtins/libm/crates/libm-test/src/run_cfg.rs index eb7e0e2c1a94..46a6a1fadaa2 100644 --- a/library/compiler-builtins/libm/crates/libm-test/src/run_cfg.rs +++ b/library/compiler-builtins/libm/crates/libm-test/src/run_cfg.rs @@ -1,13 +1,11 @@ //! Configuration for how tests get run. -#![allow(unused)] - -use std::collections::BTreeMap; use std::env; use std::sync::LazyLock; -use crate::{BaseName, FloatTy, Identifier, op}; +use crate::{BaseName, FloatTy, Identifier, test_log}; +/// The environment variable indicating which extensive tests should be run. pub const EXTENSIVE_ENV: &str = "LIBM_EXTENSIVE_TESTS"; /// Context passed to [`CheckOutput`]. @@ -49,3 +47,174 @@ pub enum CheckBasis { /// Check against infinite precision (MPFR). Mpfr, } + +/// The different kinds of generators that provide test input. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum GeneratorKind { + Domain, + Random, +} + +/// A list of all functions that should get extensive tests. +/// +/// This also supports the special test name `all` to run all tests, as well as `all_f16`, +/// `all_f32`, `all_f64`, and `all_f128` to run all tests for a specific float type. +static EXTENSIVE: LazyLock> = LazyLock::new(|| { + let var = env::var(EXTENSIVE_ENV).unwrap_or_default(); + let list = var.split(",").filter(|s| !s.is_empty()).collect::>(); + let mut ret = Vec::new(); + + let append_ty_ops = |ret: &mut Vec<_>, fty: FloatTy| { + let iter = Identifier::ALL.iter().filter(move |id| id.math_op().float_ty == fty).copied(); + ret.extend(iter); + }; + + for item in list { + match item { + "all" => ret = Identifier::ALL.to_owned(), + "all_f16" => append_ty_ops(&mut ret, FloatTy::F16), + "all_f32" => append_ty_ops(&mut ret, FloatTy::F32), + "all_f64" => append_ty_ops(&mut ret, FloatTy::F64), + "all_f128" => append_ty_ops(&mut ret, FloatTy::F128), + s => { + let id = Identifier::from_str(s) + .unwrap_or_else(|| panic!("unrecognized test name `{s}`")); + ret.push(id); + } + } + } + + ret +}); + +/// Information about the function to be tested. +#[derive(Debug)] +struct TestEnv { + /// Tests should be reduced because the platform is slow. E.g. 32-bit or emulated. + slow_platform: bool, + /// The float cannot be tested exhaustively, `f64` or `f128`. + large_float_ty: bool, + /// Env indicates that an extensive test should be run. + should_run_extensive: bool, + /// Multiprecision tests will be run. + mp_tests_enabled: bool, + /// The number of inputs to the function. + input_count: usize, +} + +impl TestEnv { + fn from_env(ctx: &CheckCtx) -> Self { + let id = ctx.fn_ident; + let op = id.math_op(); + + let will_run_mp = cfg!(feature = "test-multiprecision"); + + // Tests are pretty slow on non-64-bit targets, x86 MacOS, and targets that run in QEMU. Start + // with a reduced number on these platforms. + let slow_on_ci = crate::emulated() + || usize::BITS < 64 + || cfg!(all(target_arch = "x86_64", target_vendor = "apple")); + let slow_platform = slow_on_ci && crate::ci(); + + let large_float_ty = match op.float_ty { + FloatTy::F16 | FloatTy::F32 => false, + FloatTy::F64 | FloatTy::F128 => true, + }; + + let will_run_extensive = EXTENSIVE.contains(&id); + + let input_count = op.rust_sig.args.len(); + + Self { + slow_platform, + large_float_ty, + should_run_extensive: will_run_extensive, + mp_tests_enabled: will_run_mp, + input_count, + } + } +} + +/// The number of iterations to run for a given test. +pub fn iteration_count(ctx: &CheckCtx, gen_kind: GeneratorKind, argnum: usize) -> u64 { + let t_env = TestEnv::from_env(ctx); + + // Ideally run 5M tests + let mut domain_iter_count: u64 = 4_000_000; + + // Start with a reduced number of tests on slow platforms. + if t_env.slow_platform { + domain_iter_count = 100_000; + } + + // Larger float types get more iterations. + if t_env.large_float_ty { + domain_iter_count *= 4; + } + + // Functions with more arguments get more iterations. + let arg_multiplier = 1 << (t_env.input_count - 1); + domain_iter_count *= arg_multiplier; + + // If we will be running tests against MPFR, we don't need to test as much against musl. + // However, there are some platforms where we have to test against musl since MPFR can't be + // built. + if t_env.mp_tests_enabled && ctx.basis == CheckBasis::Musl { + domain_iter_count /= 100; + } + + // Run fewer random tests than domain tests. + let random_iter_count = domain_iter_count / 100; + + let mut total_iterations = match gen_kind { + GeneratorKind::Domain => domain_iter_count, + GeneratorKind::Random => random_iter_count, + }; + + if cfg!(optimizations_enabled) { + // Always run at least 10,000 tests. + total_iterations = total_iterations.max(10_000); + } else { + // Without optimizations, just run a quick check regardless of other parameters. + total_iterations = 800; + } + + // Adjust for the number of inputs + let ntests = match t_env.input_count { + 1 => total_iterations, + 2 => (total_iterations as f64).sqrt().ceil() as u64, + 3 => (total_iterations as f64).cbrt().ceil() as u64, + _ => panic!("test has more than three arguments"), + }; + let total = ntests.pow(t_env.input_count.try_into().unwrap()); + + test_log(&format!( + "{gen_kind:?} {basis:?} {fn_ident} arg {arg}/{args}: {ntests} iterations \ + ({total} total)", + basis = ctx.basis, + fn_ident = ctx.fn_ident, + arg = argnum + 1, + args = t_env.input_count, + )); + + ntests +} + +/// For domain tests, limit how many asymptotes or specified check points we test. +pub fn check_point_count(ctx: &CheckCtx) -> usize { + let t_env = TestEnv::from_env(ctx); + if t_env.slow_platform || !cfg!(optimizations_enabled) { 4 } else { 10 } +} + +/// When validating points of interest (e.g. asymptotes, inflection points, extremes), also check +/// this many surrounding values. +pub fn check_near_count(_ctx: &CheckCtx) -> u64 { + if cfg!(optimizations_enabled) { 100 } else { 10 } +} + +/// Check whether extensive actions should be run or skipped. +#[expect(dead_code, reason = "extensive tests have not yet been added")] +pub fn skip_extensive_test(ctx: &CheckCtx) -> bool { + let t_env = TestEnv::from_env(ctx); + !t_env.should_run_extensive +}