Auto merge of #3436 - RalfJung:win-ci, r=RalfJung

speed up Windows CI

The many-seeds test is taking 15 minutes. Let's just run that only once instead of many times on Windows.

Also refactor the CI script to make the caller control which tests are being run.

src/tools/miri/.github/workflows/ci.yml

@@ -130,6 +130,10 @@ jobs:
         run: ./miri fmt --check
       - name: clippy
         run: ./miri clippy -- -D warnings
+      - name: clippy (no features)
+        run: ./miri clippy --no-default-features -- -D warnings
+      - name: clippy (all features)
+        run: ./miri clippy --all-features -- -D warnings
       - name: rustdoc
         run: RUSTDOCFLAGS="-Dwarnings" ./miri cargo doc --document-private-items
 

src/tools/miri/ci/ci.sh

@@ -27,60 +27,59 @@ export RUSTFLAGS="-D warnings"
 export CARGO_INCREMENTAL=0
 export CARGO_EXTRA_FLAGS="--locked"
 
-# Determine configuration for installed build
+# Determine configuration for installed build (used by test-cargo-miri).
 echo "Installing release version of Miri"
-./miri install
-
-echo "Checking various feature flag configurations"
-./miri check --no-default-features # make sure this can be built
-./miri check # and this, too
-# `--all-features` is used for the build below, so no extra check needed.
+time ./miri install
 
 # Prepare debug build for direct `./miri` invocations.
 # We enable all features to make sure the Stacked Borrows consistency check runs.
 echo "Building debug version of Miri"
 export CARGO_EXTRA_FLAGS="$CARGO_EXTRA_FLAGS --all-features"
-./miri build --all-targets # the build that all the `./miri test` below will use
+time ./miri build --all-targets # the build that all the `./miri test` below will use
 
 endgroup
 
-# Test
+# Run tests. Recognizes these variables:
+# - MIRI_TEST_TARGET: the target to test. Empty for host target.
+# - GC_STRESS: if non-empty, run the GC stress test for the main test suite.
+# - MIR_OPT: if non-empty, re-run test `pass` tests with mir-opt-level=4
+# - MANY_SEEDS: if set to N, run the "many-seeds" tests N times
+# - TEST_BENCH: if non-empty, check that the benchmarks all build
+# - CARGO_MIRI_ENV: if non-empty, set some env vars and config to potentially confuse cargo-miri
 function run_tests {
-  if [ -n "${MIRI_TEST_TARGET:-}" ]; then
+  if [ -n "${MIRI_TEST_TARGET-}" ]; then
     begingroup "Testing foreign architecture $MIRI_TEST_TARGET"
   else
     begingroup "Testing host architecture"
   fi
 
   ## ui test suite
-  # On the host, also stress-test the GC.
-  if [ -z "${MIRI_TEST_TARGET:-}" ]; then
-    MIRIFLAGS="${MIRIFLAGS:-} -Zmiri-provenance-gc=1" ./miri test
+  if [ -n "${GC_STRESS-}" ]; then
+    time MIRIFLAGS="${MIRIFLAGS-} -Zmiri-provenance-gc=1" ./miri test
   else
-    ./miri test
+    time ./miri test
   fi
 
-  # Host-only tests
-  if [ -z "${MIRI_TEST_TARGET:-}" ]; then
-    # Running these on all targets is unlikely to catch more problems and would
-    # cost a lot of CI time.
-
+  ## advanced tests
+  if [ -n "${MIR_OPT-}" ]; then
     # Tests with optimizations (`-O` is what cargo passes, but crank MIR optimizations up all the
     # way, too).
     # Optimizations change diagnostics (mostly backtraces), so we don't check
     # them. Also error locations change so we don't run the failing tests.
     # We explicitly enable debug-assertions here, they are disabled by -O but we have tests
     # which exist to check that we panic on debug assertion failures.
-    MIRIFLAGS="${MIRIFLAGS:-} -O -Zmir-opt-level=4 -Cdebug-assertions=yes" MIRI_SKIP_UI_CHECKS=1 ./miri test -- tests/{pass,panic}
-
+    time MIRIFLAGS="${MIRIFLAGS-} -O -Zmir-opt-level=4 -Cdebug-assertions=yes" MIRI_SKIP_UI_CHECKS=1 ./miri test -- tests/{pass,panic}
+  fi
+  if [ -n "${MANY_SEEDS-}" ]; then
     # Also run some many-seeds tests. 64 seeds means this takes around a minute per test.
     # (Need to invoke via explicit `bash -c` for Windows.)
-    for FILE in tests/many-seeds/*.rs; do
-      MIRI_SEEDS=64 ./miri many-seeds "$BASH" -c "./miri run '$FILE'"
+    time for FILE in tests/many-seeds/*.rs; do
+      MIRI_SEEDS=$MANY_SEEDS ./miri many-seeds "$BASH" -c "./miri run '$FILE'"
     done
-
+  fi
+  if [ -n "${TEST_BENCH-}" ]; then
     # Check that the benchmarks build and run, but without actually benchmarking.
-    HYPERFINE="'$BASH' -c" ./miri bench
+    time HYPERFINE="'$BASH' -c" ./miri bench
   fi
 
   ## test-cargo-miri
@@ -91,16 +90,18 @@ function run_tests {
     PYTHON=python
   fi
   # Some environment setup that attempts to confuse the heck out of cargo-miri.
-  if [ "$HOST_TARGET" = x86_64-unknown-linux-gnu ]; then
-    # These act up on Windows (`which miri` produces a filename that does not exist?!?),
-    # so let's do this only on Linux. Also makes sure things work without these set.
-    export RUSTC=$(which rustc) # Produces a warning unless we also set MIRI
+  if [ -n "${CARGO_MIRI_ENV-}" ]; then
+    # These act up on Windows (`which miri` produces a filename that does not exist?!?).
+    # RUSTC is the main thing to set (it changes the first argument our wrapper will see).
+    # Unless MIRI is also set, that produces a warning.
+    export RUSTC=$(which rustc)
     export MIRI=$(rustc +miri --print sysroot)/bin/miri
+    # We entirely ignore other wrappers.
+    mkdir -p .cargo
+    echo 'build.rustc-wrapper = "thisdoesnotexist"' > .cargo/config.toml
   fi
-  mkdir -p .cargo
-  echo 'build.rustc-wrapper = "thisdoesnotexist"' > .cargo/config.toml
   # Run the actual test
-  ${PYTHON} test-cargo-miri/run-test.py
+  time ${PYTHON} test-cargo-miri/run-test.py
   # Clean up
   unset RUSTC MIRI
   rm -rf .cargo
@@ -109,7 +110,7 @@ function run_tests {
 }
 
 function run_tests_minimal {
-  if [ -n "${MIRI_TEST_TARGET:-}" ]; then
+  if [ -n "${MIRI_TEST_TARGET-}" ]; then
     begingroup "Testing MINIMAL foreign architecture $MIRI_TEST_TARGET: only testing $@"
   else
     echo "run_tests_minimal requires MIRI_TEST_TARGET to be set"
@@ -126,38 +127,48 @@ function run_tests_minimal {
 
 ## Main Testing Logic ##
 
-# Host target.
-run_tests
-
-# Extra targets.
 # In particular, fully cover all tier 1 targets.
 case $HOST_TARGET in
   x86_64-unknown-linux-gnu)
+    # Host
+    GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
+    # Extra tier 1
     MIRI_TEST_TARGET=i686-unknown-linux-gnu run_tests
     MIRI_TEST_TARGET=aarch64-unknown-linux-gnu run_tests
-    MIRI_TEST_TARGET=aarch64-apple-darwin run_tests
     MIRI_TEST_TARGET=i686-pc-windows-gnu run_tests
     MIRI_TEST_TARGET=x86_64-pc-windows-gnu run_tests
+    # Extra tier 2
+    MIRI_TEST_TARGET=aarch64-apple-darwin run_tests
     MIRI_TEST_TARGET=arm-unknown-linux-gnueabi run_tests
-    # Some targets are only partially supported.
+    # Partially supported targets (tier 2)
     MIRI_TEST_TARGET=x86_64-unknown-freebsd run_tests_minimal hello integer vec panic/panic concurrency/simple pthread-threadname libc-getentropy libc-getrandom libc-misc libc-fs atomic env align num_cpus
     MIRI_TEST_TARGET=i686-unknown-freebsd run_tests_minimal hello integer vec panic/panic concurrency/simple pthread-threadname libc-getentropy libc-getrandom libc-misc libc-fs atomic env align num_cpus
-
     MIRI_TEST_TARGET=aarch64-linux-android run_tests_minimal hello integer vec panic/panic
     MIRI_TEST_TARGET=wasm32-wasi run_tests_minimal no_std integer strings wasm
     MIRI_TEST_TARGET=wasm32-unknown-unknown run_tests_minimal no_std integer strings wasm
-    MIRI_TEST_TARGET=thumbv7em-none-eabihf run_tests_minimal no_std # no_std embedded architecture
-    MIRI_TEST_TARGET=tests/avr.json MIRI_NO_STD=1 run_tests_minimal no_std # JSON target file
+    MIRI_TEST_TARGET=thumbv7em-none-eabihf run_tests_minimal no_std
+    # Custom target JSON file
+    MIRI_TEST_TARGET=tests/avr.json MIRI_NO_STD=1 run_tests_minimal no_std
     ;;
   x86_64-apple-darwin)
+    # Host
+    GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=64 TEST_BENCH=1 CARGO_MIRI_ENV=1 run_tests
+    # Extra tier 1
+    MIRI_TEST_TARGET=x86_64-pc-windows-msvc CARGO_MIRI_ENV=1 run_tests
+    # Extra tier 2
     MIRI_TEST_TARGET=s390x-unknown-linux-gnu run_tests # big-endian architecture
-    MIRI_TEST_TARGET=x86_64-pc-windows-msvc run_tests
     ;;
   i686-pc-windows-msvc)
+    # Host
+    # Only smoke-test `many-seeds`; 64 runs take 15min here!
+    GC_STRESS=1 MIR_OPT=1 MANY_SEEDS=1 TEST_BENCH=1 run_tests
+    # Extra tier 1
+    # We really want to ensure a Linux target works on a Windows host,
+    # and a 64bit target works on a 32bit host.
     MIRI_TEST_TARGET=x86_64-unknown-linux-gnu run_tests
     ;;
   *)
-    echo "FATAL: unknown OS"
+    echo "FATAL: unknown host target: $HOST_TARGET"
     exit 1
     ;;
 esac

src/tools/miri/test-cargo-miri/run-test.py

@@ -31,11 +31,11 @@ def cargo_miri(cmd, quiet = True):
 
 def normalize_stdout(str):
     str = str.replace("src\\", "src/") # normalize paths across platforms
-    str = re.sub("finished in \d+\.\d\ds", "finished in $TIME", str) # the time keeps changing, obviously
+    str = re.sub("finished in \\d+\\.\\d\\ds", "finished in $TIME", str) # the time keeps changing, obviously
     return str
 
 def normalize_stderr(str):
-    str = re.sub("Preparing a sysroot for Miri \(target: [a-z0-9_-]+\)\.\.\. done\n", "", str) # remove leading cargo-miri setup output
+    str = re.sub("Preparing a sysroot for Miri \\(target: [a-z0-9_-]+\\)\\.\\.\\. done\n", "", str) # remove leading cargo-miri setup output
     return str
 
 def check_output(actual, path, name):