Rollup merge of #126265 - RalfJung:interpret-cast-validity, r=oli-obk

interpret: ensure we check bool/char for validity when they are used in a cast

In general, `Scalar::to_bits` is a bit dangerous as it bypasses all type information. We should usually prefer matching on the type and acting according to that. So I also refactored `unary_op` handling of integers to do that. The remaining `to_bits` uses are operations that just fundamentally don't care about the sign (and only work on integers).

invalid_char_cast.rs is the key new test, the others already passed before this PR.

r? `@oli-obk`

src/tools/miri/tests/fail/validity/invalid_char_cast.rs

@@ -0,0 +1,21 @@
+// Make sure we find these even with many checks disabled.
+//@compile-flags: -Zmiri-disable-alignment-check -Zmiri-disable-stacked-borrows -Zmiri-disable-validation
+#![feature(core_intrinsics)]
+#![feature(custom_mir)]
+
+use std::intrinsics::mir::*;
+
+#[custom_mir(dialect = "runtime", phase = "optimized")]
+fn cast(ptr: *const char) -> u32 {
+    mir! {
+        {
+            RET = *ptr as u32; //~ERROR: interpreting an invalid 32-bit value as a char
+            Return()
+        }
+    }
+}
+
+pub fn main() {
+    let v = u32::MAX;
+    cast(&v as *const u32 as *const char);
+}

src/tools/miri/tests/fail/validity/invalid_char_cast.stderr

@@ -0,0 +1,20 @@
+error: Undefined Behavior: interpreting an invalid 32-bit value as a char: $HEX
+  --> $DIR/invalid_char_cast.rs:LL:CC
+   |
+LL |             RET = *ptr as u32;
+   |             ^^^^^^^^^^^^^^^^^ interpreting an invalid 32-bit value as a char: $HEX
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+   = note: BACKTRACE:
+   = note: inside `cast` at $DIR/invalid_char_cast.rs:LL:CC
+note: inside `main`
+  --> $DIR/invalid_char_cast.rs:LL:CC
+   |
+LL |     cast(&v as *const u32 as *const char);
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+

src/tools/miri/tests/fail/validity/invalid_char_match.rs

@@ -0,0 +1,26 @@
+// Make sure we find these even with many checks disabled.
+//@compile-flags: -Zmiri-disable-alignment-check -Zmiri-disable-stacked-borrows -Zmiri-disable-validation
+#![feature(core_intrinsics)]
+#![feature(custom_mir)]
+
+use std::intrinsics::mir::*;
+
+#[custom_mir(dialect = "runtime", phase = "optimized")]
+fn switch_int(ptr: *const char) {
+    mir! {
+        {
+            match *ptr { //~ERROR: interpreting an invalid 32-bit value as a char
+                '0' => ret,
+                _ => ret,
+            }
+        }
+        ret = {
+            Return()
+        }
+    }
+}
+
+pub fn main() {
+    let v = u32::MAX;
+    switch_int(&v as *const u32 as *const char);
+}

src/tools/miri/tests/fail/validity/invalid_char_match.stderr

@@ -0,0 +1,23 @@
+error: Undefined Behavior: interpreting an invalid 32-bit value as a char: $HEX
+  --> $DIR/invalid_char_match.rs:LL:CC
+   |
+LL | /             match *ptr {
+LL | |                 '0' => ret,
+LL | |                 _ => ret,
+LL | |             }
+   | |_____________^ interpreting an invalid 32-bit value as a char: $HEX
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+   = note: BACKTRACE:
+   = note: inside `switch_int` at $DIR/invalid_char_match.rs:LL:CC
+note: inside `main`
+  --> $DIR/invalid_char_match.rs:LL:CC
+   |
+LL |     switch_int(&v as *const u32 as *const char);
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+

src/tools/miri/tests/fail/validity/invalid_enum_cast.rs

@@ -0,0 +1,15 @@
+// Make sure we find these even with many checks disabled.
+//@compile-flags: -Zmiri-disable-alignment-check -Zmiri-disable-stacked-borrows -Zmiri-disable-validation
+
+#[derive(Copy, Clone)]
+#[allow(unused)]
+enum E {A, B, C }
+
+fn cast(ptr: *const E) { unsafe {
+    let _val = *ptr as u32; //~ERROR: enum value has invalid tag
+}}
+
+pub fn main() {
+    let v = u32::MAX;
+    cast(&v as *const u32 as *const E);
+}

src/tools/miri/tests/fail/validity/invalid_enum_cast.stderr

@@ -0,0 +1,20 @@
+error: Undefined Behavior: enum value has invalid tag: 0xff
+  --> $DIR/invalid_enum_cast.rs:LL:CC
+   |
+LL |     let _val = *ptr as u32;
+   |                ^^^^^^^^^^^ enum value has invalid tag: 0xff
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+   = note: BACKTRACE:
+   = note: inside `cast` at $DIR/invalid_enum_cast.rs:LL:CC
+note: inside `main`
+  --> $DIR/invalid_enum_cast.rs:LL:CC
+   |
+LL |     cast(&v as *const u32 as *const E);
+   |     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to 1 previous error
+