Auto merge of #2183 - RalfJung:better-provenance-control, r=RalfJung

adjust for better provenance control

This is the Miri side of https://github.com/rust-lang/rust/pull/97684.

rust-version

@@ -1 +1 @@
-4e725bad73747a4c93a3ac53106e4b4006edc665
+9d20fd109809f20c049d6895a5be27a1fbd39daa

rustup-toolchain

@@ -30,7 +30,7 @@ else
     NEW_COMMIT="$1"
 fi
 echo "$NEW_COMMIT" > rust-version
-shift
+shift || true # don't fail if shifting fails
 
 # Check if we already are at that commit.
 CUR_COMMIT=$(rustc +miri --version -v 2>/dev/null | egrep "^commit-hash: " | cut -d " " -f 2)

src/diagnostics.rs

@@ -229,9 +229,16 @@ pub fn report_error<'tcx, 'mir>(
             };
             #[rustfmt::skip]
             let helps = match e.kind() {
-                Unsupported(UnsupportedOpInfo::ThreadLocalStatic(_) | UnsupportedOpInfo::ReadExternStatic(_)) =>
+                Unsupported(
+                    UnsupportedOpInfo::ThreadLocalStatic(_) |
+                    UnsupportedOpInfo::ReadExternStatic(_)
+                ) =>
                     panic!("Error should never be raised by Miri: {:?}", e.kind()),
-                Unsupported(_) =>
+                Unsupported(
+                    UnsupportedOpInfo::Unsupported(_) |
+                    UnsupportedOpInfo::PartialPointerOverwrite(_) |
+                    UnsupportedOpInfo::ReadPointerAsBytes
+                ) =>
                     vec![(None, format!("this is likely not a bug in the program; it indicates that the program performed an operation that the interpreter does not support"))],
                 UndefinedBehavior(UndefinedBehaviorInfo::AlignmentCheckFailed { .. })
                     if ecx.machine.check_alignment == AlignmentCheck::Symbolic
@@ -245,7 +252,8 @@ pub fn report_error<'tcx, 'mir>(
                         (None, format!("this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior")),
                         (None, format!("see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information")),
                     ],
-                _ => vec![],
+                InvalidProgram(_) | ResourceExhaustion(_) | MachineStop(_) =>
+                    vec![],
             };
             (Some(title), helps)
         }

src/helpers.rs

@@ -681,7 +681,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
             // FIXME: We are re-getting the allocation each time around the loop.
             // Would be nice if we could somehow "extend" an existing AllocRange.
             let alloc = this.get_ptr_alloc(ptr.offset(len, this)?, size1, Align::ONE)?.unwrap(); // not a ZST, so we will get a result
-            let byte = alloc.read_scalar(alloc_range(Size::ZERO, size1))?.to_u8()?;
+            let byte = alloc.read_integer(Size::ZERO, size1)?.to_u8()?;
             if byte == 0 {
                 break;
             } else {
@@ -703,7 +703,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx
             // FIXME: We are re-getting the allocation each time around the loop.
             // Would be nice if we could somehow "extend" an existing AllocRange.
             let alloc = this.get_ptr_alloc(ptr, size2, align2)?.unwrap(); // not a ZST, so we will get a result
-            let wchar = alloc.read_scalar(alloc_range(Size::ZERO, size2))?.to_u16()?;
+            let wchar = alloc.read_integer(Size::ZERO, size2)?.to_u16()?;
             if wchar == 0 {
                 break;
             } else {

tests/fail/branchless-select-i128-pointer.rs

@@ -9,10 +9,10 @@ fn main() {
     for &my_bool in &[true, false] {
         let mask = -(my_bool as TwoPtrs); // false -> 0, true -> -1 aka !0
         // This is branchless code to select one or the other pointer.
-        // For now, Miri brafs on it, but if this code ever passes we better make sure it behaves correctly.
+        // However, it drops provenance when transmuting to TwoPtrs, so this is UB.
         let val = unsafe {
-            transmute::<_, &str>(
-                !mask & transmute::<_, TwoPtrs>("false !") | mask & transmute::<_, TwoPtrs>("true !"), //~ERROR encountered (potentially part of) a pointer, but expected plain (non-pointer) bytes
+            transmute::<_, &str>( //~ERROR type validation failed: encountered a dangling reference
+                !mask & transmute::<_, TwoPtrs>("false !") | mask & transmute::<_, TwoPtrs>("true !"),
             )
         };
         println!("{}", val);

tests/fail/branchless-select-i128-pointer.stderr

@@ -1,8 +1,10 @@
-error: Undefined Behavior: type validation failed: encountered (potentially part of) a pointer, but expected plain (non-pointer) bytes
+error: Undefined Behavior: type validation failed: encountered a dangling reference (address $HEX is unallocated)
   --> $DIR/branchless-select-i128-pointer.rs:LL:CC
    |
-LL |                 !mask & transmute::<_, TwoPtrs>("false !") | mask & transmute::<_, TwoPtrs>("true !"),
-   |                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered (potentially part of) a pointer, but expected plain (non-pointer) bytes
+LL | /             transmute::<_, &str>(
+LL | |                 !mask & transmute::<_, TwoPtrs>("false !") | mask & transmute::<_, TwoPtrs>("true !"),
+LL | |             )
+   | |_____________^ type validation failed: encountered a dangling reference (address $HEX is unallocated)
    |
    = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
    = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information

tests/fail/pointer_partial_overwrite.rs

@@ -2,7 +2,7 @@
 // compile-flags: -Zmiri-disable-alignment-check -Zmiri-disable-stacked-borrows -Zmiri-disable-validation
 
 // Test what happens when we overwrite parts of a pointer.
-// Also see <https://github.com/rust-lang/rust/issues/87184>.
+// Also see <https://github.com/rust-lang/miri/issues/2181>.
 
 fn main() {
     let mut p = &42;

tests/fail/pointer_partial_read.rs

@@ -1,9 +0,0 @@
-// Test what happens when we read parts of a pointer.
-// Related to <https://github.com/rust-lang/rust/issues/69488>.
-fn main() {
-    let x = 13;
-    let y = &x;
-    let z = &y as *const &i32 as *const u8;
-    // the deref fails, because we are reading only a part of the pointer
-    let _val = unsafe { *z }; //~ ERROR unable to turn pointer into raw bytes
-}

tests/fail/pointer_partial_read.stderr

@@ -1,14 +0,0 @@
-error: unsupported operation: unable to turn pointer into raw bytes
-  --> $DIR/pointer_partial_read.rs:LL:CC
-   |
-LL |     let _val = unsafe { *z };
-   |                         ^^ unable to turn pointer into raw bytes
-   |
-   = help: this is likely not a bug in the program; it indicates that the program performed an operation that the interpreter does not support
-           
-   = note: inside `main` at $DIR/pointer_partial_read.rs:LL:CC
-
-note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
-
-error: aborting due to previous error
-

tests/fail/provenance/permissive_provenance_transmute.rs

@@ -0,0 +1,27 @@
+// compile-flags: -Zmiri-permissive-provenance -Zmiri-disable-stacked-borrows
+#![feature(strict_provenance)]
+
+use std::mem;
+
+// This is the example from
+// <https://github.com/rust-lang/unsafe-code-guidelines/issues/286#issuecomment-1085144431>.
+
+unsafe fn deref(left: *const u8, right: *const u8) {
+    let left_int: usize = mem::transmute(left);
+    let right_int: usize = mem::transmute(right);
+    if left_int == right_int {
+        // The compiler is allowed to replace `left_int` by `right_int` here...
+        let left_ptr: *const u8 = mem::transmute(left_int);
+        // ...which however means here it could be dereferencing the wrong pointer.
+        let _val = *left_ptr; //~ERROR dereferencing pointer failed
+    }
+}
+
+fn main() {
+    let ptr1 = &0u8 as *const u8;
+    let ptr2 = &1u8 as *const u8;
+    unsafe {
+        // Two pointers with the same address but different provenance.
+        deref(ptr1, ptr2.with_addr(ptr1.addr()));
+    }
+}

tests/fail/provenance/permissive_provenance_transmute.stderr

@@ -0,0 +1,20 @@
+error: Undefined Behavior: dereferencing pointer failed: $HEX is not a valid pointer
+  --> $DIR/permissive_provenance_transmute.rs:LL:CC
+   |
+LL |         let _val = *left_ptr;
+   |                    ^^^^^^^^^ dereferencing pointer failed: $HEX is not a valid pointer
+   |
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
+           
+   = note: inside `deref` at $DIR/permissive_provenance_transmute.rs:LL:CC
+note: inside `main` at $DIR/permissive_provenance_transmute.rs:LL:CC
+  --> $DIR/permissive_provenance_transmute.rs:LL:CC
+   |
+LL |         deref(ptr1, ptr2.with_addr(ptr1.addr()));
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
+
+error: aborting due to previous error
+

tests/fail/provenance/ptr_int_unexposed.rs

@@ -1,12 +1,12 @@
-// compile-flags: -Zmiri-permissive-provenance -Zmiri-disable-stacked-borrows -Zmiri-allow-ptr-int-transmute
+// compile-flags: -Zmiri-permissive-provenance -Zmiri-disable-stacked-borrows
+#![feature(strict_provenance)]
 
 fn main() {
     let x: i32 = 3;
     let x_ptr = &x as *const i32;
 
-    // TODO: switch this to addr() once we intrinsify it
-    let x_usize: usize = unsafe { std::mem::transmute(x_ptr) };
-    // Cast back a pointer that did *not* get exposed.
-    let ptr = x_usize as *const i32;
+    let x_usize: usize = x_ptr.addr();
+    // Cast back an address that did *not* get exposed.
+    let ptr = std::ptr::from_exposed_addr::<i32>(x_usize);
     assert_eq!(unsafe { *ptr }, 3); //~ ERROR Undefined Behavior: dereferencing pointer failed
 }

tests/fail/provenance/strict_provenance_transmute.rs

@@ -7,13 +7,13 @@ use std::mem;
 // <https://github.com/rust-lang/unsafe-code-guidelines/issues/286#issuecomment-1085144431>.
 
 unsafe fn deref(left: *const u8, right: *const u8) {
-    let left_int: usize = mem::transmute(left); //~ERROR expected plain (non-pointer) bytes
+    let left_int: usize = mem::transmute(left);
     let right_int: usize = mem::transmute(right);
     if left_int == right_int {
         // The compiler is allowed to replace `left_int` by `right_int` here...
         let left_ptr: *const u8 = mem::transmute(left_int);
         // ...which however means here it could be dereferencing the wrong pointer.
-        let _val = *left_ptr;
+        let _val = *left_ptr; //~ERROR dereferencing pointer failed
     }
 }
 

tests/fail/provenance/strict_provenance_transmute.stderr

@@ -1,8 +1,8 @@
-error: Undefined Behavior: type validation failed: encountered pointer to $HEX[ALLOC]<TAG>, but expected plain (non-pointer) bytes
+error: Undefined Behavior: dereferencing pointer failed: $HEX is not a valid pointer
   --> $DIR/strict_provenance_transmute.rs:LL:CC
    |
-LL |     let left_int: usize = mem::transmute(left);
-   |                           ^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered pointer to $HEX[ALLOC]<TAG>, but expected plain (non-pointer) bytes
+LL |         let _val = *left_ptr;
+   |                    ^^^^^^^^^ dereferencing pointer failed: $HEX is not a valid pointer
    |
    = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
    = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information

tests/fail/stacked_borrows/illegal_read3.rs

@@ -1,16 +1,18 @@
-// compile-flags: -Zmiri-allow-ptr-int-transmute
 // A callee may not read the destination of our `&mut` without us noticing.
 // Thise code got carefully checked to not introduce any reborrows
 // that are not explicit in the source. Let's hope the compiler does not break this later!
 
-#![feature(untagged_unions)]
-
 use std::mem;
 
+union HiddenRef {
+    // We avoid retagging at this type, so shared vs mutable does not matter.
+    r: &'static i32,
+}
+
 fn main() {
     let mut x: i32 = 15;
     let xref1 = &mut x;
-    let xref1_sneaky: usize = unsafe { mem::transmute_copy(&xref1) };
+    let xref1_sneaky: HiddenRef = unsafe { mem::transmute_copy(&xref1) };
     // Derived from `xref1`, so using raw value is still ok, ...
     let xref2 = &mut *xref1;
     callee(xref1_sneaky);
@@ -19,14 +21,8 @@ fn main() {
     //~^ ERROR: borrow stack
 }
 
-fn callee(xref1: usize) {
-    // Transmuting through a union to avoid retagging.
-    union UsizeToRef {
-        from: usize,
-        to: &'static mut i32,
-    }
-    let xref1 = UsizeToRef { from: xref1 };
+fn callee(xref1: HiddenRef) {
     // Doing the deref and the transmute (through the union) in the same place expression
     // should avoid retagging.
-    let _val = unsafe { *xref1.to };
+    let _val = unsafe { *xref1.r };
 }

tests/fail/stacked_borrows/illegal_read3.stderr

@@ -17,8 +17,8 @@ LL |     let xref2 = &mut *xref1;
 help: <TAG> was later invalidated at offsets [0x0..0x4]
   --> $DIR/illegal_read3.rs:LL:CC
    |
-LL |     let _val = unsafe { *xref1.to };
-   |                         ^^^^^^^^^
+LL |     let _val = unsafe { *xref1.r };
+   |                         ^^^^^^^^
    = note: inside `main` at $DIR/illegal_read3.rs:LL:CC
 
 note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace

tests/fail/transmute-pair-uninit.rs

@@ -1,4 +1,3 @@
-// compile-flags: -Zmiri-allow-uninit-numbers
 #![feature(core_intrinsics)]
 
 use std::mem;
@@ -18,6 +17,6 @@ fn main() {
         assert_eq!(byte, 0);
     }
     let v = unsafe { *z.offset(first_undef) };
+    //~^ ERROR uninitialized
     if v == 0 { println!("it is zero"); }
-    //~^ ERROR this operation requires initialized memory
 }

tests/fail/transmute-pair-uninit.stderr

@@ -1,8 +1,8 @@
-error: Undefined Behavior: using uninitialized data, but this operation requires initialized memory
+error: Undefined Behavior: type validation failed: encountered uninitialized bytes, but expected initialized bytes
   --> $DIR/transmute-pair-uninit.rs:LL:CC
    |
-LL |     if v == 0 { println!("it is zero"); }
-   |        ^^^^^^ using uninitialized data, but this operation requires initialized memory
+LL |     let v = unsafe { *z.offset(first_undef) };
+   |                      ^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered uninitialized bytes, but expected initialized bytes
    |
    = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
    = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information

tests/fail/transmute_fat1.rs

@@ -1,5 +1,5 @@
-// This should fail even without validation
-// compile-flags: -Zmiri-disable-validation
+// error-pattern: type validation failed: encountered a pointer
+// normalize-stderr-test: "\[u8; (08|16)\]" -> "$$ARRAY"
 
 fn main() {
     #[cfg(target_pointer_width="64")]
@@ -8,7 +8,7 @@ fn main() {
     };
     #[cfg(target_pointer_width="32")]
     let bad = unsafe {
-        std::mem::transmute::<&[u8], [u8; 8]>(&[1u8])
+        std::mem::transmute::<&[u8], [u8; 08]>(&[1u8])
     };
-    let _val = bad[0] + bad[bad.len()-1]; //~ ERROR unable to turn pointer into raw bytes
+    let _val = bad[0] + bad[bad.len()-1];
 }

tests/fail/transmute_fat1.stderr

@@ -1,10 +1,11 @@
-error: unsupported operation: unable to turn pointer into raw bytes
+error: Undefined Behavior: type validation failed: encountered a pointer, but expected plain (non-pointer) bytes
   --> $DIR/transmute_fat1.rs:LL:CC
    |
-LL |     let _val = bad[0] + bad[bad.len()-1];
-   |                ^^^^^^ unable to turn pointer into raw bytes
+LL |         std::mem::transmute::<&[u8], $ARRAY>(&[1u8])
+   |         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered a pointer, but expected plain (non-pointer) bytes
    |
-   = help: this is likely not a bug in the program; it indicates that the program performed an operation that the interpreter does not support
+   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
+   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
            
    = note: inside `main` at $DIR/transmute_fat1.rs:LL:CC
 

tests/fail/uninit_byte_read.rs

@@ -1,7 +1,6 @@
-// compile-flags: -Zmiri-allow-uninit-numbers
 fn main() {
     let v: Vec<u8> = Vec::with_capacity(10);
-    let undef = unsafe { *v.get_unchecked(5) };
-    let x = undef + 1; //~ ERROR this operation requires initialized memory
+    let undef = unsafe { *v.get_unchecked(5) }; //~ ERROR uninitialized
+    let x = undef + 1;
     panic!("this should never print: {}", x);
 }

tests/fail/uninit_byte_read.stderr

@@ -1,8 +1,8 @@
-error: Undefined Behavior: using uninitialized data, but this operation requires initialized memory
+error: Undefined Behavior: type validation failed: encountered uninitialized bytes, but expected initialized bytes
   --> $DIR/uninit_byte_read.rs:LL:CC
    |
-LL |     let x = undef + 1;
-   |             ^^^^^^^^^ using uninitialized data, but this operation requires initialized memory
+LL |     let undef = unsafe { *v.get_unchecked(5) };
+   |                          ^^^^^^^^^^^^^^^^^^^ type validation failed: encountered uninitialized bytes, but expected initialized bytes
    |
    = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
    = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information

tests/fail/validity/invalid_enum_tag_256variants_uninit.rs

@@ -1,3 +1,4 @@
+// Even when uninit numbers are allowed, this enum is not.
 // compile-flags: -Zmiri-allow-uninit-numbers
 #![allow(unused, deprecated, invalid_value)]
 

tests/fail/validity/ptr_integer_transmute.rs

@@ -1,4 +0,0 @@
-fn main() {
-    let r = &mut 42;
-    let _i: usize = unsafe { std::mem::transmute(r) }; //~ ERROR expected plain (non-pointer) bytes
-}

tests/fail/validity/ptr_integer_transmute.stderr

@@ -1,15 +0,0 @@
-error: Undefined Behavior: type validation failed: encountered pointer to $HEX[ALLOC]<TAG>, but expected plain (non-pointer) bytes
-  --> $DIR/ptr_integer_transmute.rs:LL:CC
-   |
-LL |     let _i: usize = unsafe { std::mem::transmute(r) };
-   |                              ^^^^^^^^^^^^^^^^^^^^^^ type validation failed: encountered pointer to $HEX[ALLOC]<TAG>, but expected plain (non-pointer) bytes
-   |
-   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
-   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
-           
-   = note: inside `main` at $DIR/ptr_integer_transmute.rs:LL:CC
-
-note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
-
-error: aborting due to previous error
-

tests/pass/intptrcast.rs

@@ -1,6 +1,4 @@
-// compile-flags: -Zmiri-allow-ptr-int-transmute
-
-// This returns a miri pointer at type usize, if the argument is a proper pointer
+// This strips provenance
 fn transmute_ptr_to_int<T>(x: *const T) -> usize {
     unsafe { std::mem::transmute(x) }
 }
@@ -39,7 +37,7 @@ fn transmute() {
     // transmuting.
     let a: *const i32 = &42;
     let b = transmute_ptr_to_int(a) as u8;
-    let c = a as usize as u8;
+    let c = a as u8;
     assert_eq!(b, c);
 }
 

tests/pass/ptr_int_transmute.rs

@@ -0,0 +1,22 @@
+// Test what happens when we read parts of a pointer.
+// Related to <https://github.com/rust-lang/rust/issues/69488>.
+fn ptr_partial_read() {
+    let x = 13;
+    let y = &x;
+    let z = &y as *const &i32 as *const u8;
+
+    // This just strips provenance, but should work fine otherwise.
+    let _val = unsafe { *z };
+}
+
+fn transmute_strip_provenance() {
+    let r = &mut 42;
+    let addr = r as *mut _ as usize;
+    let i: usize = unsafe { std::mem::transmute(r) };
+    assert_eq!(i, addr);
+}
+
+fn main() {
+    ptr_partial_read();
+    transmute_strip_provenance();
+}

tests/pass/transmute_fat.rs

@@ -1,14 +1,11 @@
 // Stacked Borrows disallows this becuase the reference is never cast to a raw pointer.
-// compile-flags: -Zmiri-disable-stacked-borrows -Zmiri-allow-ptr-int-transmute
+// compile-flags: -Zmiri-disable-stacked-borrows
 
 fn main() {
     // If we are careful, we can exploit data layout...
     let raw = unsafe {
-        std::mem::transmute::<&[u8], [usize; 2]>(&[42])
+        std::mem::transmute::<&[u8], [*const u8; 2]>(&[42])
     };
-    let ptr = raw[0] + raw[1];
-    // We transmute both ways, to really test allow-ptr-int-transmute.
-    let ptr: *const u8 = unsafe { std::mem::transmute(ptr) };
-    // The pointer is one-past-the end, but we decrement it into bounds before using it
-    assert_eq!(unsafe { *ptr.offset(-1) }, 42);
+    let ptr: *const u8 = unsafe { std::mem::transmute_copy(&raw) };
+    assert_eq!(unsafe { *ptr }, 42);
 }