diff --git a/cargo-miri/bin.rs b/cargo-miri/bin.rs
index c4ed92d03852..233d81826eec 100644
--- a/cargo-miri/bin.rs
+++ b/cargo-miri/bin.rs
@@ -374,12 +374,15 @@ fn setup(subcommand: MiriCommand) {
         }
         None => {
             // Check for `rust-src` rustup component.
-            let sysroot = miri()
-                .args(&["--print", "sysroot"])
-                .output()
-                .expect("failed to determine sysroot")
-                .stdout;
-            let sysroot = std::str::from_utf8(&sysroot).unwrap();
+            let output =
+                miri().args(&["--print", "sysroot"]).output().expect("failed to determine sysroot");
+            if !output.status.success() {
+                show_error(format!(
+                    "Failed to determine sysroot; Miri said:\n{}",
+                    String::from_utf8_lossy(&output.stderr).trim_end()
+                ));
+            }
+            let sysroot = std::str::from_utf8(&output.stdout).unwrap();
             let sysroot = Path::new(sysroot.trim_end_matches('\n'));
             // Check for `$SYSROOT/lib/rustlib/src/rust/library`; test if that contains `std/Cargo.toml`.
             let rustup_src =
diff --git a/src/bin/miri.rs b/src/bin/miri.rs
index 0f464da175d0..8bd33b591d72 100644
--- a/src/bin/miri.rs
+++ b/src/bin/miri.rs
@@ -1,5 +1,5 @@
 #![feature(rustc_private, stmt_expr_attributes)]
-#![allow(clippy::manual_range_contains)]
+#![allow(clippy::manual_range_contains, clippy::useless_format)]
 
 extern crate rustc_data_structures;
 extern crate rustc_driver;
@@ -143,6 +143,11 @@ impl rustc_driver::Callbacks for MiriBeRustCompilerCalls {
     }
 }
 
+fn show_error(msg: String) -> ! {
+    eprintln!("fatal error: {}", msg);
+    std::process::exit(1)
+}
+
 fn init_early_loggers() {
     // Note that our `extern crate log` is *not* the same as rustc's; as a result, we have to
     // initialize them both, and we always initialize `miri`'s first.
@@ -214,13 +219,26 @@ fn compile_time_sysroot() -> Option<String> {
     let home = option_env!("RUSTUP_HOME").or(option_env!("MULTIRUST_HOME"));
     let toolchain = option_env!("RUSTUP_TOOLCHAIN").or(option_env!("MULTIRUST_TOOLCHAIN"));
     Some(match (home, toolchain) {
-        (Some(home), Some(toolchain)) => format!("{}/toolchains/{}", home, toolchain),
-        _ =>
-            option_env!("RUST_SYSROOT")
-                .expect(
+        (Some(home), Some(toolchain)) => {
+            // Check that at runtime, we are still in this toolchain.
+            let toolchain_runtime =
+                env::var_os("RUSTUP_TOOLCHAIN").or_else(|| env::var_os("MULTIRUST_TOOLCHAIN"));
+            if !matches!(toolchain_runtime, Some(r) if r == toolchain) {
+                show_error(format!(
+                    "This Miri got built with local toolchain `{toolchain}`, but now is being run under a different toolchain. \n\
+                    Make sure to run Miri in the toolchain it got built with, e.g. via `cargo +{toolchain} miri`."
+                ));
+            }
+
+            format!("{}/toolchains/{}", home, toolchain)
+        }
+        _ => option_env!("RUST_SYSROOT")
+            .unwrap_or_else(|| {
+                show_error(format!(
                     "To build Miri without rustup, set the `RUST_SYSROOT` env var at build time",
-                )
-                .to_owned(),
+                ))
+            })
+            .to_owned(),
     })
 }
 
diff --git a/src/diagnostics.rs b/src/diagnostics.rs
index 4dfa7bd07cbe..1c6cfa096863 100644
--- a/src/diagnostics.rs
+++ b/src/diagnostics.rs
@@ -505,4 +505,22 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
             }
         });
     }
+
+    /// We had a panic in Miri itself, try to print something useful.
+    fn handle_ice(&self) {
+        eprintln!();
+        eprintln!(
+            "Miri caused an ICE during evaluation. Here's the interpreter backtrace at the time of the panic:"
+        );
+        let this = self.eval_context_ref();
+        let stacktrace = this.generate_stacktrace();
+        report_msg(
+            this,
+            DiagLevel::Note,
+            "the place in the program where the ICE was triggered",
+            vec![],
+            vec![],
+            &stacktrace,
+        );
+    }
 }
diff --git a/src/eval.rs b/src/eval.rs
index 0ab85ef264d9..44459621e393 100644
--- a/src/eval.rs
+++ b/src/eval.rs
@@ -1,7 +1,10 @@
 //! Main evaluator loop and setting up the initial stack frame.
 
+use std::collections::HashSet;
 use std::ffi::OsStr;
 use std::iter;
+use std::panic::{self, AssertUnwindSafe};
+use std::thread;
 
 use log::info;
 
@@ -15,8 +18,6 @@ use rustc_target::spec::abi::Abi;
 
 use rustc_session::config::EntryFnType;
 
-use std::collections::HashSet;
-
 use crate::*;
 
 #[derive(Copy, Clone, Debug, PartialEq)]
@@ -326,7 +327,7 @@ pub fn eval_entry<'tcx>(
     };
 
     // Perform the main execution.
-    let res: InterpResult<'_, i64> = (|| {
+    let res: thread::Result<InterpResult<'_, i64>> = panic::catch_unwind(AssertUnwindSafe(|| {
         // Main loop.
         loop {
             let info = ecx.preprocess_diagnostics();
@@ -356,7 +357,11 @@ pub fn eval_entry<'tcx>(
         }
         let return_code = ecx.read_scalar(&ret_place.into())?.to_machine_isize(&ecx)?;
         Ok(return_code)
-    })();
+    }));
+    let res = res.unwrap_or_else(|panic_payload| {
+        ecx.handle_ice();
+        panic::resume_unwind(panic_payload)
+    });
 
     // Machine cleanup.
     // Execution of the program has halted so any memory access we do here