From 5d1e92d132208bb7347c22ac9c40577c1f285c28 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 31 Mar 2024 09:36:40 +0200 Subject: [PATCH 1/7] ci: separetely control which host-only tests are run where No functional change in this commit --- src/tools/miri/ci/ci.sh | 54 +++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/src/tools/miri/ci/ci.sh b/src/tools/miri/ci/ci.sh index cccf10a7d703..8827cb2b60e9 100755 --- a/src/tools/miri/ci/ci.sh +++ b/src/tools/miri/ci/ci.sh @@ -44,7 +44,12 @@ export CARGO_EXTRA_FLAGS="$CARGO_EXTRA_FLAGS --all-features" endgroup -# Test +# Run tests. Recognizes these variables: +# - MIRI_TEST_TARGET: the target to test. Empty for host target. +# - GC_STRESS: if non-empty, run the GC stress test for the main test suite. +# - MIR_OPT: if non-empty, re-run test `pass` tests with mir-opt-level=4 +# - MANY_SEEDS: if set to N, run the "many-seeds" tests N times +# - TEST_BENCH: if non-empty, check that the benchmarks all build function run_tests { if [ -n "${MIRI_TEST_TARGET:-}" ]; then begingroup "Testing foreign architecture $MIRI_TEST_TARGET" @@ -53,18 +58,14 @@ function run_tests { fi ## ui test suite - # On the host, also stress-test the GC. - if [ -z "${MIRI_TEST_TARGET:-}" ]; then + if [ -n "${GC_STRESS:-}" ]; then MIRIFLAGS="${MIRIFLAGS:-} -Zmiri-provenance-gc=1" ./miri test else ./miri test fi - # Host-only tests - if [ -z "${MIRI_TEST_TARGET:-}" ]; then - # Running these on all targets is unlikely to catch more problems and would - # cost a lot of CI time. - + ## advanced tests + if [ -n "${MIR_OPT:-}" ]; then # Tests with optimizations (`-O` is what cargo passes, but crank MIR optimizations up all the # way, too). # Optimizations change diagnostics (mostly backtraces), so we don't check @@ -72,13 +73,15 @@ function run_tests { # We explicitly enable debug-assertions here, they are disabled by -O but we have tests # which exist to check that we panic on debug assertion failures. MIRIFLAGS="${MIRIFLAGS:-} -O -Zmir-opt-level=4 -Cdebug-assertions=yes" MIRI_SKIP_UI_CHECKS=1 ./miri test -- tests/{pass,panic} - + fi + if [ -n "${MANY_SEEDS:-}" ]; then # Also run some many-seeds tests. 64 seeds means this takes around a minute per test. # (Need to invoke via explicit `bash -c` for Windows.) for FILE in tests/many-seeds/*.rs; do - MIRI_SEEDS=64 ./miri many-seeds "$BASH" -c "./miri run '$FILE'" + MIRI_SEEDS=$MANY_SEEDS ./miri many-seeds "$BASH" -c "./miri run '$FILE'" done - + fi + if [ -n "${TEST_BENCH:-}" ]; then # Check that the benchmarks build and run, but without actually benchmarking. HYPERFINE="'$BASH' -c" ./miri bench fi @@ -126,34 +129,43 @@ function run_tests_minimal { ## Main Testing Logic ## -# Host target. -run_tests - -# Extra targets. # In particular, fully cover all tier 1 targets. case $HOST_TARGET in x86_64-unknown-linux-gnu) + # Host + GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 run_tests + # Extra tier 1 MIRI_TEST_TARGET=i686-unknown-linux-gnu run_tests MIRI_TEST_TARGET=aarch64-unknown-linux-gnu run_tests - MIRI_TEST_TARGET=aarch64-apple-darwin run_tests MIRI_TEST_TARGET=i686-pc-windows-gnu run_tests MIRI_TEST_TARGET=x86_64-pc-windows-gnu run_tests + # Extra tier 2 + MIRI_TEST_TARGET=aarch64-apple-darwin run_tests MIRI_TEST_TARGET=arm-unknown-linux-gnueabi run_tests - # Some targets are only partially supported. + # Partially supported targets (tier 2) MIRI_TEST_TARGET=x86_64-unknown-freebsd run_tests_minimal hello integer vec panic/panic concurrency/simple pthread-threadname libc-getentropy libc-getrandom libc-misc libc-fs atomic env align num_cpus MIRI_TEST_TARGET=i686-unknown-freebsd run_tests_minimal hello integer vec panic/panic concurrency/simple pthread-threadname libc-getentropy libc-getrandom libc-misc libc-fs atomic env align num_cpus - MIRI_TEST_TARGET=aarch64-linux-android run_tests_minimal hello integer vec panic/panic MIRI_TEST_TARGET=wasm32-wasi run_tests_minimal no_std integer strings wasm MIRI_TEST_TARGET=wasm32-unknown-unknown run_tests_minimal no_std integer strings wasm - MIRI_TEST_TARGET=thumbv7em-none-eabihf run_tests_minimal no_std # no_std embedded architecture - MIRI_TEST_TARGET=tests/avr.json MIRI_NO_STD=1 run_tests_minimal no_std # JSON target file + MIRI_TEST_TARGET=thumbv7em-none-eabihf run_tests_minimal no_std + # Custom target JSON file + MIRI_TEST_TARGET=tests/avr.json MIRI_NO_STD=1 run_tests_minimal no_std ;; x86_64-apple-darwin) - MIRI_TEST_TARGET=s390x-unknown-linux-gnu run_tests # big-endian architecture + # Host + GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 run_tests + # Extra tier 1 MIRI_TEST_TARGET=x86_64-pc-windows-msvc run_tests + # Extra tier 2 + MIRI_TEST_TARGET=s390x-unknown-linux-gnu run_tests # big-endian architecture ;; i686-pc-windows-msvc) + # Host + GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 run_tests + # Extra tier 1 + # We really want to ensure a Linux target works on a Windows host, + # and a 64bit target works on a 32bit host. MIRI_TEST_TARGET=x86_64-unknown-linux-gnu run_tests ;; *) From 9395e1fa6fa60dd6cee4c5f37e7c649332346500 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 31 Mar 2024 09:44:55 +0200 Subject: [PATCH 2/7] bash: use variable expansion that properly distinguishes null vs non-existing --- src/tools/miri/ci/ci.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/tools/miri/ci/ci.sh b/src/tools/miri/ci/ci.sh index 8827cb2b60e9..5c93cc7feedb 100755 --- a/src/tools/miri/ci/ci.sh +++ b/src/tools/miri/ci/ci.sh @@ -51,37 +51,37 @@ endgroup # - MANY_SEEDS: if set to N, run the "many-seeds" tests N times # - TEST_BENCH: if non-empty, check that the benchmarks all build function run_tests { - if [ -n "${MIRI_TEST_TARGET:-}" ]; then + if [ -n "${MIRI_TEST_TARGET-}" ]; then begingroup "Testing foreign architecture $MIRI_TEST_TARGET" else begingroup "Testing host architecture" fi ## ui test suite - if [ -n "${GC_STRESS:-}" ]; then - MIRIFLAGS="${MIRIFLAGS:-} -Zmiri-provenance-gc=1" ./miri test + if [ -n "${GC_STRESS-}" ]; then + MIRIFLAGS="${MIRIFLAGS-} -Zmiri-provenance-gc=1" ./miri test else ./miri test fi ## advanced tests - if [ -n "${MIR_OPT:-}" ]; then + if [ -n "${MIR_OPT-}" ]; then # Tests with optimizations (`-O` is what cargo passes, but crank MIR optimizations up all the # way, too). # Optimizations change diagnostics (mostly backtraces), so we don't check # them. Also error locations change so we don't run the failing tests. # We explicitly enable debug-assertions here, they are disabled by -O but we have tests # which exist to check that we panic on debug assertion failures. - MIRIFLAGS="${MIRIFLAGS:-} -O -Zmir-opt-level=4 -Cdebug-assertions=yes" MIRI_SKIP_UI_CHECKS=1 ./miri test -- tests/{pass,panic} + MIRIFLAGS="${MIRIFLAGS-} -O -Zmir-opt-level=4 -Cdebug-assertions=yes" MIRI_SKIP_UI_CHECKS=1 ./miri test -- tests/{pass,panic} fi - if [ -n "${MANY_SEEDS:-}" ]; then + if [ -n "${MANY_SEEDS-}" ]; then # Also run some many-seeds tests. 64 seeds means this takes around a minute per test. # (Need to invoke via explicit `bash -c` for Windows.) for FILE in tests/many-seeds/*.rs; do MIRI_SEEDS=$MANY_SEEDS ./miri many-seeds "$BASH" -c "./miri run '$FILE'" done fi - if [ -n "${TEST_BENCH:-}" ]; then + if [ -n "${TEST_BENCH-}" ]; then # Check that the benchmarks build and run, but without actually benchmarking. HYPERFINE="'$BASH' -c" ./miri bench fi @@ -112,7 +112,7 @@ function run_tests { } function run_tests_minimal { - if [ -n "${MIRI_TEST_TARGET:-}" ]; then + if [ -n "${MIRI_TEST_TARGET-}" ]; then begingroup "Testing MINIMAL foreign architecture $MIRI_TEST_TARGET: only testing $@" else echo "run_tests_minimal requires MIRI_TEST_TARGET to be set" @@ -169,7 +169,7 @@ case $HOST_TARGET in MIRI_TEST_TARGET=x86_64-unknown-linux-gnu run_tests ;; *) - echo "FATAL: unknown OS" + echo "FATAL: unknown host target: $HOST_TARGET" exit 1 ;; esac From 836ea71d05584645a8ba340c5236dc0e7aa66226 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 31 Mar 2024 09:45:38 +0200 Subject: [PATCH 3/7] windows ci: run many-seeds test less often --- src/tools/miri/ci/ci.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tools/miri/ci/ci.sh b/src/tools/miri/ci/ci.sh index 5c93cc7feedb..e14bc56cb78e 100755 --- a/src/tools/miri/ci/ci.sh +++ b/src/tools/miri/ci/ci.sh @@ -162,7 +162,8 @@ case $HOST_TARGET in ;; i686-pc-windows-msvc) # Host - GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 run_tests + # Only smoke-test `many-seeds`; 64 runs take 15min here! + GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=1 TEST_BENCH=1 run_tests # Extra tier 1 # We really want to ensure a Linux target works on a Windows host, # and a 64bit target works on a 32bit host. From e9ee19004e1b5a412cce369838f26114886d9f48 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 31 Mar 2024 09:50:49 +0200 Subject: [PATCH 4/7] also control the cargo-miri env var test separately --- src/tools/miri/ci/ci.sh | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/tools/miri/ci/ci.sh b/src/tools/miri/ci/ci.sh index e14bc56cb78e..4e94ca41b7ff 100755 --- a/src/tools/miri/ci/ci.sh +++ b/src/tools/miri/ci/ci.sh @@ -50,6 +50,7 @@ endgroup # - MIR_OPT: if non-empty, re-run test `pass` tests with mir-opt-level=4 # - MANY_SEEDS: if set to N, run the "many-seeds" tests N times # - TEST_BENCH: if non-empty, check that the benchmarks all build +# - CARGO_MIRI_ENV: if non-empty, set some env vars and config to potentially confuse cargo-miri function run_tests { if [ -n "${MIRI_TEST_TARGET-}" ]; then begingroup "Testing foreign architecture $MIRI_TEST_TARGET" @@ -94,14 +95,16 @@ function run_tests { PYTHON=python fi # Some environment setup that attempts to confuse the heck out of cargo-miri. - if [ "$HOST_TARGET" = x86_64-unknown-linux-gnu ]; then - # These act up on Windows (`which miri` produces a filename that does not exist?!?), - # so let's do this only on Linux. Also makes sure things work without these set. - export RUSTC=$(which rustc) # Produces a warning unless we also set MIRI + if [ -n "${CARGO_MIRI_ENV-}" ]; then + # These act up on Windows (`which miri` produces a filename that does not exist?!?). + # RUSTC is the main thing to set (it changes the first argument our wrapper will see). + # Unless MIRI is also set, that produces a warning. + export RUSTC=$(which rustc) export MIRI=$(rustc +miri --print sysroot)/bin/miri + # We entirely ignore other wrappers. + mkdir -p .cargo + echo 'build.rustc-wrapper = "thisdoesnotexist"' > .cargo/config.toml fi - mkdir -p .cargo - echo 'build.rustc-wrapper = "thisdoesnotexist"' > .cargo/config.toml # Run the actual test ${PYTHON} test-cargo-miri/run-test.py # Clean up @@ -133,7 +136,7 @@ function run_tests_minimal { case $HOST_TARGET in x86_64-unknown-linux-gnu) # Host - GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 run_tests + GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests # Extra tier 1 MIRI_TEST_TARGET=i686-unknown-linux-gnu run_tests MIRI_TEST_TARGET=aarch64-unknown-linux-gnu run_tests @@ -154,9 +157,9 @@ case $HOST_TARGET in ;; x86_64-apple-darwin) # Host - GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 run_tests + GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests # Extra tier 1 - MIRI_TEST_TARGET=x86_64-pc-windows-msvc run_tests + MIRI_TEST_TARGET=x86_64-pc-windows-msvc CARGO_MIRI_ENV=1 run_tests # Extra tier 2 MIRI_TEST_TARGET=s390x-unknown-linux-gnu run_tests # big-endian architecture ;; From 540f2411e6f9f039b5e4cd4369bfddf0a0479731 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 31 Mar 2024 10:32:29 +0200 Subject: [PATCH 5/7] show the time that the main CI steps take --- src/tools/miri/ci/ci.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/tools/miri/ci/ci.sh b/src/tools/miri/ci/ci.sh index 4e94ca41b7ff..2cd918fc2b15 100755 --- a/src/tools/miri/ci/ci.sh +++ b/src/tools/miri/ci/ci.sh @@ -60,9 +60,9 @@ function run_tests { ## ui test suite if [ -n "${GC_STRESS-}" ]; then - MIRIFLAGS="${MIRIFLAGS-} -Zmiri-provenance-gc=1" ./miri test + time MIRIFLAGS="${MIRIFLAGS-} -Zmiri-provenance-gc=1" ./miri test else - ./miri test + time ./miri test fi ## advanced tests @@ -73,18 +73,18 @@ function run_tests { # them. Also error locations change so we don't run the failing tests. # We explicitly enable debug-assertions here, they are disabled by -O but we have tests # which exist to check that we panic on debug assertion failures. - MIRIFLAGS="${MIRIFLAGS-} -O -Zmir-opt-level=4 -Cdebug-assertions=yes" MIRI_SKIP_UI_CHECKS=1 ./miri test -- tests/{pass,panic} + time MIRIFLAGS="${MIRIFLAGS-} -O -Zmir-opt-level=4 -Cdebug-assertions=yes" MIRI_SKIP_UI_CHECKS=1 ./miri test -- tests/{pass,panic} fi if [ -n "${MANY_SEEDS-}" ]; then # Also run some many-seeds tests. 64 seeds means this takes around a minute per test. # (Need to invoke via explicit `bash -c` for Windows.) - for FILE in tests/many-seeds/*.rs; do + time for FILE in tests/many-seeds/*.rs; do MIRI_SEEDS=$MANY_SEEDS ./miri many-seeds "$BASH" -c "./miri run '$FILE'" done fi if [ -n "${TEST_BENCH-}" ]; then # Check that the benchmarks build and run, but without actually benchmarking. - HYPERFINE="'$BASH' -c" ./miri bench + time HYPERFINE="'$BASH' -c" ./miri bench fi ## test-cargo-miri @@ -106,7 +106,7 @@ function run_tests { echo 'build.rustc-wrapper = "thisdoesnotexist"' > .cargo/config.toml fi # Run the actual test - ${PYTHON} test-cargo-miri/run-test.py + time ${PYTHON} test-cargo-miri/run-test.py # Clean up unset RUSTC MIRI rm -rf .cargo From 660a5713f527baaa4da3a400e15d83b677ed02fb Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 31 Mar 2024 10:37:59 +0200 Subject: [PATCH 6/7] python: fix regex backslash escapes --- src/tools/miri/test-cargo-miri/run-test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tools/miri/test-cargo-miri/run-test.py b/src/tools/miri/test-cargo-miri/run-test.py index 21479bc4d58e..cac11dff7755 100755 --- a/src/tools/miri/test-cargo-miri/run-test.py +++ b/src/tools/miri/test-cargo-miri/run-test.py @@ -31,11 +31,11 @@ def cargo_miri(cmd, quiet = True): def normalize_stdout(str): str = str.replace("src\\", "src/") # normalize paths across platforms - str = re.sub("finished in \d+\.\d\ds", "finished in $TIME", str) # the time keeps changing, obviously + str = re.sub("finished in \\d+\\.\\d\\ds", "finished in $TIME", str) # the time keeps changing, obviously return str def normalize_stderr(str): - str = re.sub("Preparing a sysroot for Miri \(target: [a-z0-9_-]+\)\.\.\. done\n", "", str) # remove leading cargo-miri setup output + str = re.sub("Preparing a sysroot for Miri \\(target: [a-z0-9_-]+\\)\\.\\.\\. done\n", "", str) # remove leading cargo-miri setup output return str def check_output(actual, path, name): From a8edf26a61158c5074ce121dd3c65d5a9509eb38 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Sun, 31 Mar 2024 10:44:18 +0200 Subject: [PATCH 7/7] move './miri check' for features to the style check job --- src/tools/miri/.github/workflows/ci.yml | 4 ++++ src/tools/miri/ci/ci.sh | 11 +++-------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/tools/miri/.github/workflows/ci.yml b/src/tools/miri/.github/workflows/ci.yml index 2c5868d5b37a..c5979e18a3b5 100644 --- a/src/tools/miri/.github/workflows/ci.yml +++ b/src/tools/miri/.github/workflows/ci.yml @@ -130,6 +130,10 @@ jobs: run: ./miri fmt --check - name: clippy run: ./miri clippy -- -D warnings + - name: clippy (no features) + run: ./miri clippy --no-default-features -- -D warnings + - name: clippy (all features) + run: ./miri clippy --all-features -- -D warnings - name: rustdoc run: RUSTDOCFLAGS="-Dwarnings" ./miri cargo doc --document-private-items diff --git a/src/tools/miri/ci/ci.sh b/src/tools/miri/ci/ci.sh index 2cd918fc2b15..32b58547d7c2 100755 --- a/src/tools/miri/ci/ci.sh +++ b/src/tools/miri/ci/ci.sh @@ -27,20 +27,15 @@ export RUSTFLAGS="-D warnings" export CARGO_INCREMENTAL=0 export CARGO_EXTRA_FLAGS="--locked" -# Determine configuration for installed build +# Determine configuration for installed build (used by test-cargo-miri). echo "Installing release version of Miri" -./miri install - -echo "Checking various feature flag configurations" -./miri check --no-default-features # make sure this can be built -./miri check # and this, too -# `--all-features` is used for the build below, so no extra check needed. +time ./miri install # Prepare debug build for direct `./miri` invocations. # We enable all features to make sure the Stacked Borrows consistency check runs. echo "Building debug version of Miri" export CARGO_EXTRA_FLAGS="$CARGO_EXTRA_FLAGS --all-features" -./miri build --all-targets # the build that all the `./miri test` below will use +time ./miri build --all-targets # the build that all the `./miri test` below will use endgroup