diff --git a/library/compiler-builtins/libm/.github/workflows/main.yml b/library/compiler-builtins/libm/.github/workflows/main.yml
index f0c5fe7c03e5..bfd86497b4ec 100644
--- a/library/compiler-builtins/libm/.github/workflows/main.yml
+++ b/library/compiler-builtins/libm/.github/workflows/main.yml
@@ -125,7 +125,7 @@ jobs:
     - uses: Swatinem/rust-cache@v2
     - name: Download musl source
       run: ./ci/download-musl.sh
-    - run: cargo bench --all
+    - run: cargo bench --all --features libm-test/short-benchmarks,libm-test/build-musl
 
   msrv:
     name: Check MSRV
diff --git a/library/compiler-builtins/libm/Cargo.toml b/library/compiler-builtins/libm/Cargo.toml
index 2e74012eae44..1786277664a1 100644
--- a/library/compiler-builtins/libm/Cargo.toml
+++ b/library/compiler-builtins/libm/Cargo.toml
@@ -38,7 +38,6 @@ force-soft-floats = []
 resolver = "2"
 members = [
   "crates/compiler-builtins-smoke-test",
-  "crates/libm-bench",
   "crates/libm-macros",
   "crates/libm-test",
   "crates/musl-math-sys",
diff --git a/library/compiler-builtins/libm/ci/run.sh b/library/compiler-builtins/libm/ci/run.sh
index 9f642326ba82..a211bc98cbf9 100755
--- a/library/compiler-builtins/libm/ci/run.sh
+++ b/library/compiler-builtins/libm/ci/run.sh
@@ -83,4 +83,8 @@ else
     # unstable with a feature
     $cmd --features "unstable-intrinsics"
     $cmd --release --features "unstable-intrinsics"
+
+    # Make sure benchmarks have correct results
+    $cmd --benches
+    $cmd --benches --release
 fi
diff --git a/library/compiler-builtins/libm/crates/libm-bench/Cargo.toml b/library/compiler-builtins/libm/crates/libm-bench/Cargo.toml
deleted file mode 100644
index ee8c5820037f..000000000000
--- a/library/compiler-builtins/libm/crates/libm-bench/Cargo.toml
+++ /dev/null
@@ -1,16 +0,0 @@
-[package]
-name = "libm-bench"
-version = "0.1.0"
-authors = ["Gonzalo Brito Gadeschi <gonzalobg88@gmail.com>"]
-edition = "2021"
-license = "MIT OR Apache-2.0"
-publish = false
-
-[dependencies]
-libm = { path = "../..", default-features = false }
-rand = "0.8.5"
-paste = "1.0.15"
-
-[features]
-default = []
-unstable = [ "libm/unstable" ]
diff --git a/library/compiler-builtins/libm/crates/libm-bench/benches/bench.rs b/library/compiler-builtins/libm/crates/libm-bench/benches/bench.rs
deleted file mode 100644
index ca999b90f27c..000000000000
--- a/library/compiler-builtins/libm/crates/libm-bench/benches/bench.rs
+++ /dev/null
@@ -1,116 +0,0 @@
-#![feature(test)]
-extern crate test;
-
-use rand::Rng;
-use test::Bencher;
-
-macro_rules! unary {
-  ($($func:ident),*) => ($(
-      paste::item! {
-        #[bench]
-        pub fn [<$func>](bh: &mut Bencher) {
-            let mut rng = rand::thread_rng();
-            let x = rng.gen::<f64>();
-            bh.iter(|| test::black_box(libm::[<$func>](x)))
-        }
-        #[bench]
-        pub fn [<$func f>](bh: &mut Bencher) {
-            let mut rng = rand::thread_rng();
-            let x = rng.gen::<f32>();
-            bh.iter(|| test::black_box(libm::[<$func f>](x)))
-        }
-    }
-  )*);
-}
-macro_rules! binary {
-  ($($func:ident),*) => ($(
-      paste::item! {
-        #[bench]
-        pub fn [<$func>](bh: &mut Bencher) {
-            let mut rng = rand::thread_rng();
-            let x = rng.gen::<f64>();
-            let y = rng.gen::<f64>();
-            bh.iter(|| test::black_box(libm::[<$func>](x, y)))
-        }
-        #[bench]
-        pub fn [<$func f>](bh: &mut Bencher) {
-            let mut rng = rand::thread_rng();
-            let x = rng.gen::<f32>();
-            let y = rng.gen::<f32>();
-            bh.iter(|| test::black_box(libm::[<$func f>](x, y)))
-        }
-    }
-  )*);
-  ($($func:ident);*) => ($(
-      paste::item! {
-        #[bench]
-        pub fn [<$func>](bh: &mut Bencher) {
-            let mut rng = rand::thread_rng();
-            let x = rng.gen::<f64>();
-            let n = rng.gen::<i32>();
-            bh.iter(|| test::black_box(libm::[<$func>](x, n)))
-        }
-        #[bench]
-        pub fn [<$func f>](bh: &mut Bencher) {
-            let mut rng = rand::thread_rng();
-            let x = rng.gen::<f32>();
-            let n = rng.gen::<i32>();
-            bh.iter(|| test::black_box(libm::[<$func f>](x, n)))
-        }
-    }
-  )*);
-}
-macro_rules! trinary {
-  ($($func:ident),*) => ($(
-      paste::item! {
-        #[bench]
-        pub fn [<$func>](bh: &mut Bencher) {
-            let mut rng = rand::thread_rng();
-            let x = rng.gen::<f64>();
-            let y = rng.gen::<f64>();
-            let z = rng.gen::<f64>();
-            bh.iter(|| test::black_box(libm::[<$func>](x, y, z)))
-        }
-        #[bench]
-        pub fn [<$func f>](bh: &mut Bencher) {
-            let mut rng = rand::thread_rng();
-            let x = rng.gen::<f32>();
-            let y = rng.gen::<f32>();
-            let z = rng.gen::<f32>();
-            bh.iter(|| test::black_box(libm::[<$func f>](x, y, z)))
-        }
-    }
-  )*);
-}
-macro_rules! bessel {
-  ($($func:ident),*) => ($(
-      paste::item! {
-        #[bench]
-        pub fn [<$func>](bh: &mut Bencher) {
-            let mut rng = rand::thread_rng();
-            let mut n = rng.gen::<i32>();
-            n &= 0xffff;
-            let x = rng.gen::<f64>();
-            bh.iter(|| test::black_box(libm::[<$func>](n, x)))
-        }
-        #[bench]
-        pub fn [<$func f>](bh: &mut Bencher) {
-            let mut rng = rand::thread_rng();
-            let mut n = rng.gen::<i32>();
-            n &= 0xffff;
-            let x = rng.gen::<f32>();
-            bh.iter(|| test::black_box(libm::[<$func f>](n, x)))
-        }
-    }
-  )*);
-}
-
-unary!(
-    acos, acosh, asin, atan, cbrt, ceil, cos, cosh, erf, exp, exp2, exp10, expm1, fabs, floor, j0,
-    j1, lgamma, log, log1p, log2, log10, rint, round, sin, sinh, sqrt, tan, tanh, tgamma, trunc,
-    y0, y1
-);
-binary!(atan2, copysign, fdim, fmax, fmin, fmod, hypot, pow);
-trinary!(fma);
-bessel!(jn, yn);
-binary!(ldexp; scalbn);
diff --git a/library/compiler-builtins/libm/crates/libm-test/Cargo.toml b/library/compiler-builtins/libm/crates/libm-test/Cargo.toml
index 72ac57232b3d..fedf745ed7f4 100644
--- a/library/compiler-builtins/libm/crates/libm-test/Cargo.toml
+++ b/library/compiler-builtins/libm/crates/libm-test/Cargo.toml
@@ -15,6 +15,12 @@ test-multiprecision = ["dep:az", "dep:rug"]
 # Build our own musl for testing and benchmarks
 build-musl = ["dep:musl-math-sys"]
 
+# Enable report generation without bringing in more dependencies by default
+benchmarking-reports = ["criterion/plotters", "criterion/html_reports"]
+
+# Run with a reduced set of benchmarks, such as for CI
+short-benchmarks = []
+
 [dependencies]
 anyhow = "1.0.90"
 az = { version = "1.2.1", optional = true }
@@ -32,3 +38,10 @@ getrandom = { version = "0.2", features = ["js"] }
 
 [build-dependencies]
 rand = { version = "0.8.5", optional = true }
+
+[dev-dependencies]
+criterion = { version = "0.5.1", default-features = false, features = ["cargo_bench_support"] }
+
+[[bench]]
+name = "random"
+harness = false
diff --git a/library/compiler-builtins/libm/crates/libm-test/benches/random.rs b/library/compiler-builtins/libm/crates/libm-test/benches/random.rs
new file mode 100644
index 000000000000..6c9047c3cdb4
--- /dev/null
+++ b/library/compiler-builtins/libm/crates/libm-test/benches/random.rs
@@ -0,0 +1,119 @@
+use std::hint::black_box;
+use std::time::Duration;
+
+use criterion::{Criterion, criterion_main};
+use libm_test::gen::random;
+use libm_test::{CheckBasis, CheckCtx, TupleCall};
+
+/// Benchmark with this many items to get a variety
+const BENCH_ITER_ITEMS: usize = if cfg!(feature = "short-benchmarks") { 50 } else { 500 };
+
+macro_rules! musl_rand_benches {
+    (
+        fn_name: $fn_name:ident,
+        CFn: $CFn:ty,
+        CArgs: $CArgs:ty,
+        CRet: $CRet:ty,
+        RustFn: $RustFn:ty,
+        RustArgs: $RustArgs:ty,
+        RustRet: $RustRet:ty,
+        fn_extra: $skip_on_i586:expr,
+    ) => {
+        paste::paste! {
+            fn [< musl_bench_ $fn_name >](c: &mut Criterion) {
+                let fn_name = stringify!($fn_name);
+
+                let ulp = libm_test::musl_allowed_ulp(fn_name);
+                let ctx = CheckCtx::new(ulp, fn_name, CheckBasis::Musl);
+                let benchvec: Vec<_> = random::get_test_cases::<$RustArgs>(&ctx)
+                    .take(BENCH_ITER_ITEMS)
+                    .collect();
+
+                // Perform a sanity check that we are benchmarking the same thing
+                // Don't test against musl if it is not available
+                #[cfg(feature = "build-musl")]
+                for input in benchvec.iter().copied() {
+                    use anyhow::Context;
+                    use libm_test::{CheckBasis, CheckCtx, CheckOutput};
+
+                    if cfg!(x86_no_sse) && $skip_on_i586 {
+                        break;
+                    }
+
+                    let musl_res = input.call(musl_math_sys::$fn_name as $CFn);
+                    let crate_res = input.call(libm::$fn_name as $RustFn);
+
+                    let ctx = CheckCtx::new(ulp, fn_name, CheckBasis::Musl);
+                    crate_res.validate(musl_res, input, &ctx).context(fn_name).unwrap();
+                }
+
+                /* Function pointers are black boxed to avoid inlining in the benchmark loop */
+
+                let mut group = c.benchmark_group(fn_name);
+                group.bench_function("crate", |b| b.iter(|| {
+                    let f = black_box(libm::$fn_name as $RustFn);
+                    for input in benchvec.iter().copied() {
+                        input.call(f);
+                    }
+                }));
+
+                // Don't test against musl if it is not available
+                #[cfg(feature = "build-musl")]
+                group.bench_function("musl", |b| b.iter(|| {
+                    let f = black_box(musl_math_sys::$fn_name as $CFn);
+                    for input in benchvec.iter().copied() {
+                        input.call(f);
+                    }
+                }));
+            }
+        }
+    };
+}
+
+libm_macros::for_each_function! {
+    callback: musl_rand_benches,
+    skip: [],
+    fn_extra: match MACRO_FN_NAME {
+        // FIXME(correctness): wrong result on i586
+        exp10 | exp10f | exp2 | exp2f => true,
+        _ => false
+    }
+}
+
+macro_rules! run_callback {
+    (
+        fn_name: $fn_name:ident,
+        CFn: $_CFn:ty,
+        CArgs: $_CArgs:ty,
+        CRet: $_CRet:ty,
+        RustFn: $_RustFn:ty,
+        RustArgs: $_RustArgs:ty,
+        RustRet: $_RustRet:ty,
+        extra: [$criterion:ident],
+    ) => {
+        paste::paste! {
+            [< musl_bench_ $fn_name >](&mut $criterion)
+        }
+    };
+}
+
+pub fn musl_random() {
+    let mut criterion = Criterion::default();
+
+    // For CI, run a short 0.5s warmup and 1.0s tests. This makes benchmarks complete in
+    // about the same time as other tests.
+    if cfg!(feature = "short-benchmarks") {
+        criterion = criterion
+            .warm_up_time(Duration::from_millis(500))
+            .measurement_time(Duration::from_millis(1000));
+    }
+
+    criterion = criterion.configure_from_args();
+
+    libm_macros::for_each_function! {
+        callback: run_callback,
+        extra: [criterion],
+    };
+}
+
+criterion_main!(musl_random);