From f4b61ba509e71710df5c14ac282fbdd512344072 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Wed, 14 Jul 2021 22:10:17 +0200 Subject: [PATCH] adjustions and cleanup to make Miri build again --- .../rustc_middle/src/mir/interpret/pointer.rs | 14 ++- .../rustc_middle/src/mir/interpret/value.rs | 54 +-------- compiler/rustc_middle/src/ty/consts/int.rs | 11 +- compiler/rustc_mir/src/const_eval/machine.rs | 2 +- compiler/rustc_mir/src/const_eval/mod.rs | 2 +- compiler/rustc_mir/src/interpret/cast.rs | 4 +- compiler/rustc_mir/src/interpret/intern.rs | 6 +- .../rustc_mir/src/interpret/intrinsics.rs | 12 +- compiler/rustc_mir/src/interpret/machine.rs | 108 ++++++++---------- compiler/rustc_mir/src/interpret/memory.rs | 73 ++++++------ compiler/rustc_mir/src/interpret/place.rs | 42 ++++--- compiler/rustc_mir/src/interpret/step.rs | 5 +- .../rustc_mir/src/transform/const_prop.rs | 2 +- 13 files changed, 134 insertions(+), 201 deletions(-) diff --git a/compiler/rustc_middle/src/mir/interpret/pointer.rs b/compiler/rustc_middle/src/mir/interpret/pointer.rs index a95e39e18113..307d7d284620 100644 --- a/compiler/rustc_middle/src/mir/interpret/pointer.rs +++ b/compiler/rustc_middle/src/mir/interpret/pointer.rs @@ -171,7 +171,7 @@ impl From> for Pointer> { } impl Pointer> { - pub fn into_pointer_or_offset(self) -> Result, Size> { + pub fn into_pointer_or_addr(self) -> Result, Size> { match self.provenance { Some(tag) => Ok(Pointer::new(tag, self.offset)), None => Err(self.offset), @@ -187,6 +187,13 @@ impl Pointer> { } } +impl Pointer> { + #[inline(always)] + pub fn null() -> Self { + Pointer { provenance: None, offset: Size::ZERO } + } +} + impl<'tcx, Tag> Pointer { #[inline(always)] pub fn new(provenance: Tag, offset: Size) -> Self { @@ -206,9 +213,14 @@ impl<'tcx, Tag> Pointer { where Tag: Provenance, { + // FIXME: This is wrong! `self.offset` might be an absolute address. Pointer { offset: self.offset, provenance: self.provenance.erase_for_fmt() } } + pub fn map_provenance(self, f: impl FnOnce(Tag) -> Tag) -> Self { + Pointer { provenance: f(self.provenance), ..self } + } + #[inline] pub fn offset(self, i: Size, cx: &impl HasDataLayout) -> InterpResult<'tcx, Self> { Ok(Pointer { diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs index bb6f1bb21c62..1be04f8c18b1 100644 --- a/compiler/rustc_middle/src/mir/interpret/value.rs +++ b/compiler/rustc_middle/src/mir/interpret/value.rs @@ -6,7 +6,7 @@ use rustc_apfloat::{ Float, }; use rustc_macros::HashStable; -use rustc_target::abi::{HasDataLayout, Size, TargetDataLayout}; +use rustc_target::abi::{HasDataLayout, Size}; use crate::ty::{Lift, ParamEnv, ScalarInt, Ty, TyCtxt}; @@ -179,7 +179,7 @@ impl From for Scalar { } } -impl<'tcx, Tag> Scalar { +impl Scalar { pub const ZST: Self = Scalar::Int(ScalarInt::ZST); #[inline(always)] @@ -202,56 +202,6 @@ impl<'tcx, Tag> Scalar { Scalar::Int(ScalarInt::null(cx.pointer_size())) } - #[inline(always)] - fn ptr_op( - self, - dl: &TargetDataLayout, - f_int: impl FnOnce(u64) -> InterpResult<'tcx, u64>, - f_ptr: impl FnOnce(Pointer) -> InterpResult<'tcx, Pointer>, - ) -> InterpResult<'tcx, Self> { - match self { - Scalar::Int(int) => Ok(Scalar::Int(int.ptr_sized_op(dl, f_int)?)), - Scalar::Ptr(ptr, sz) => { - debug_assert_eq!(u64::from(sz), dl.pointer_size().bytes()); - Ok(Scalar::Ptr(f_ptr(ptr)?, sz)) - } - } - } - - #[inline] - pub fn ptr_offset(self, i: Size, cx: &impl HasDataLayout) -> InterpResult<'tcx, Self> { - let dl = cx.data_layout(); - self.ptr_op(dl, |int| dl.offset(int, i.bytes()), |ptr| ptr.offset(i, dl)) - } - - #[inline] - pub fn ptr_wrapping_offset(self, i: Size, cx: &impl HasDataLayout) -> Self { - let dl = cx.data_layout(); - self.ptr_op( - dl, - |int| Ok(dl.overflowing_offset(int, i.bytes()).0), - |ptr| Ok(ptr.wrapping_offset(i, dl)), - ) - .unwrap() - } - - #[inline] - pub fn ptr_signed_offset(self, i: i64, cx: &impl HasDataLayout) -> InterpResult<'tcx, Self> { - let dl = cx.data_layout(); - self.ptr_op(dl, |int| dl.signed_offset(int, i), |ptr| ptr.signed_offset(i, dl)) - } - - #[inline] - pub fn ptr_wrapping_signed_offset(self, i: i64, cx: &impl HasDataLayout) -> Self { - let dl = cx.data_layout(); - self.ptr_op( - dl, - |int| Ok(dl.overflowing_signed_offset(int, i).0), - |ptr| Ok(ptr.wrapping_signed_offset(i, dl)), - ) - .unwrap() - } - #[inline] pub fn from_bool(b: bool) -> Self { Scalar::Int(b.into()) diff --git a/compiler/rustc_middle/src/ty/consts/int.rs b/compiler/rustc_middle/src/ty/consts/int.rs index 8ed8ea6a0bc5..8262bc261996 100644 --- a/compiler/rustc_middle/src/ty/consts/int.rs +++ b/compiler/rustc_middle/src/ty/consts/int.rs @@ -1,7 +1,7 @@ use rustc_apfloat::ieee::{Double, Single}; use rustc_apfloat::Float; use rustc_serialize::{Decodable, Decoder, Encodable, Encoder}; -use rustc_target::abi::{Size, TargetDataLayout}; +use rustc_target::abi::Size; use std::convert::{TryFrom, TryInto}; use std::fmt; @@ -193,15 +193,6 @@ impl ScalarInt { self.data == 0 } - pub(crate) fn ptr_sized_op( - self, - dl: &TargetDataLayout, - f_int: impl FnOnce(u64) -> Result, - ) -> Result { - assert_eq!(u64::from(self.size), dl.pointer_size.bytes()); - Ok(Self::try_from_uint(f_int(u64::try_from(self.data).unwrap())?, self.size()).unwrap()) - } - #[inline] pub fn try_from_uint(i: impl Into, size: Size) -> Option { let data = i.into(); diff --git a/compiler/rustc_mir/src/const_eval/machine.rs b/compiler/rustc_mir/src/const_eval/machine.rs index 2bebbc65c24c..c809f4f273aa 100644 --- a/compiler/rustc_mir/src/const_eval/machine.rs +++ b/compiler/rustc_mir/src/const_eval/machine.rs @@ -312,7 +312,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, align, interpret::MemoryKind::Machine(MemoryKind::Heap), )?; - ecx.write_scalar(Scalar::from_pointer(ptr, &*ecx.tcx), dest)?; + ecx.write_pointer(ptr, dest)?; } _ => { return Err(ConstEvalErrKind::NeedsRfc(format!( diff --git a/compiler/rustc_mir/src/const_eval/mod.rs b/compiler/rustc_mir/src/const_eval/mod.rs index 78124428787c..a334165df4cb 100644 --- a/compiler/rustc_mir/src/const_eval/mod.rs +++ b/compiler/rustc_mir/src/const_eval/mod.rs @@ -35,7 +35,7 @@ pub(crate) fn const_caller_location( if intern_const_alloc_recursive(&mut ecx, InternKind::Constant, &loc_place).is_err() { bug!("intern_const_alloc_recursive should not error in this case") } - ConstValue::Scalar(Scalar::from_pointer(loc_place.ptr.into_pointer_or_offset().unwrap(), &tcx)) + ConstValue::Scalar(Scalar::from_pointer(loc_place.ptr.into_pointer_or_addr().unwrap(), &tcx)) } /// Convert an evaluated constant to a type level constant diff --git a/compiler/rustc_mir/src/interpret/cast.rs b/compiler/rustc_mir/src/interpret/cast.rs index ca7fd7010f3d..514c1aa9646a 100644 --- a/compiler/rustc_mir/src/interpret/cast.rs +++ b/compiler/rustc_mir/src/interpret/cast.rs @@ -57,7 +57,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { .ok_or_else(|| err_inval!(TooGeneric))?; let fn_ptr = self.memory.create_fn_alloc(FnVal::Instance(instance)); - self.write_scalar(Scalar::from_pointer(fn_ptr, &*self.tcx), dest)?; + self.write_pointer(fn_ptr, dest)?; } _ => span_bug!(self.cur_span(), "reify fn pointer on {:?}", src.layout.ty), } @@ -88,7 +88,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { ty::ClosureKind::FnOnce, ); let fn_ptr = self.memory.create_fn_alloc(FnVal::Instance(instance)); - self.write_scalar(Scalar::from_pointer(fn_ptr, &*self.tcx), dest)?; + self.write_pointer(fn_ptr, dest)?; } _ => span_bug!(self.cur_span(), "closure fn pointer on {:?}", src.layout.ty), } diff --git a/compiler/rustc_mir/src/interpret/intern.rs b/compiler/rustc_mir/src/interpret/intern.rs index b0606aba4164..f2457d11d9ee 100644 --- a/compiler/rustc_mir/src/interpret/intern.rs +++ b/compiler/rustc_mir/src/interpret/intern.rs @@ -23,7 +23,7 @@ use rustc_middle::ty::{self, layout::TyAndLayout, Ty}; use rustc_ast::Mutability; -use super::{AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, ValueVisitor}; +use super::{AllocId, Allocation, InterpCx, MPlaceTy, Machine, MemoryKind, PlaceTy, ValueVisitor}; use crate::const_eval; pub trait CompileTimeMachine<'mir, 'tcx, T> = Machine< @@ -425,11 +425,11 @@ impl<'mir, 'tcx: 'mir, M: super::intern::CompileTimeMachine<'mir, 'tcx, !>> layout: TyAndLayout<'tcx>, f: impl FnOnce( &mut InterpCx<'mir, 'tcx, M>, - &MPlaceTy<'tcx, M::PointerTag>, + &PlaceTy<'tcx, M::PointerTag>, ) -> InterpResult<'tcx, ()>, ) -> InterpResult<'tcx, &'tcx Allocation> { let dest = self.allocate(layout, MemoryKind::Stack)?; - f(self, &dest)?; + f(self, &dest.into())?; let mut alloc = self.memory.alloc_map.remove(&dest.ptr.provenance.unwrap()).unwrap().1; alloc.mutability = Mutability::Not; Ok(self.tcx.intern_const_alloc(alloc)) diff --git a/compiler/rustc_mir/src/interpret/intrinsics.rs b/compiler/rustc_mir/src/interpret/intrinsics.rs index 4e2db2f13f17..9335b783c775 100644 --- a/compiler/rustc_mir/src/interpret/intrinsics.rs +++ b/compiler/rustc_mir/src/interpret/intrinsics.rs @@ -337,17 +337,17 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let pointee_ty = substs.type_at(0); let offset_ptr = self.ptr_offset_inbounds(ptr, pointee_ty, offset_count)?; - self.write_scalar(Scalar::from_maybe_pointer(offset_ptr, self), dest)?; + self.write_pointer(offset_ptr, dest)?; } sym::arith_offset => { - let ptr = self.read_scalar(&args[0])?.check_init()?; + let ptr = self.read_pointer(&args[0])?; let offset_count = self.read_scalar(&args[1])?.to_machine_isize(self)?; let pointee_ty = substs.type_at(0); let pointee_size = i64::try_from(self.layout_of(pointee_ty)?.size.bytes()).unwrap(); let offset_bytes = offset_count.wrapping_mul(pointee_size); - let offset_ptr = ptr.ptr_wrapping_signed_offset(offset_bytes, self); - self.write_scalar(offset_ptr, dest)?; + let offset_ptr = ptr.wrapping_signed_offset(offset_bytes, self); + self.write_pointer(offset_ptr, dest)?; } sym::ptr_offset_from => { let a = self.read_immediate(&args[0])?.to_scalar()?; @@ -379,8 +379,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { // General case: we need two pointers. let a = self.scalar_to_ptr(a); let b = self.scalar_to_ptr(b); - let (a_alloc_id, a_offset, _) = self.memory.ptr_force_alloc(a)?; - let (b_alloc_id, b_offset, _) = self.memory.ptr_force_alloc(b)?; + let (a_alloc_id, a_offset, _) = self.memory.ptr_get_alloc(a)?; + let (b_alloc_id, b_offset, _) = self.memory.ptr_get_alloc(b)?; if a_alloc_id != b_alloc_id { throw_ub_format!( "ptr_offset_from cannot compute offset of pointers into different \ diff --git a/compiler/rustc_mir/src/interpret/machine.rs b/compiler/rustc_mir/src/interpret/machine.rs index c135e4f99634..7b8f2aecd0d4 100644 --- a/compiler/rustc_mir/src/interpret/machine.rs +++ b/compiler/rustc_mir/src/interpret/machine.rs @@ -7,14 +7,14 @@ use std::fmt::Debug; use std::hash::Hash; use rustc_middle::mir; -use rustc_middle::ty::{self, Ty}; +use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::def_id::DefId; use rustc_target::abi::Size; use rustc_target::spec::abi::Abi; use super::{ - AllocId, Allocation, Frame, ImmTy, InterpCx, InterpResult, LocalValue, MemPlace, Memory, - MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar, StackPopUnwind, + AllocId, AllocRange, Allocation, Frame, ImmTy, InterpCx, InterpResult, LocalValue, MemPlace, + Memory, MemoryKind, OpTy, Operand, PlaceTy, Pointer, Provenance, Scalar, StackPopUnwind, }; /// Data returned by Machine::stack_pop, @@ -262,34 +262,40 @@ pub trait Machine<'mir, 'tcx>: Sized { } /// Return the `AllocId` for the given thread-local static in the current thread. - fn thread_local_static_alloc_id( + fn thread_local_static_base_pointer( _ecx: &mut InterpCx<'mir, 'tcx, Self>, def_id: DefId, - ) -> InterpResult<'tcx, AllocId> { + ) -> InterpResult<'tcx, Pointer> { throw_unsup!(ThreadLocalStatic(def_id)) } - /// Return the `AllocId` backing the given `extern static`. - fn extern_static_alloc_id( + /// Return the root pointer for the given `extern static`. + fn extern_static_base_pointer( mem: &Memory<'mir, 'tcx, Self>, def_id: DefId, - ) -> InterpResult<'tcx, AllocId> { - // Use the `AllocId` associated with the `DefId`. Any actual *access* will fail. - Ok(mem.tcx.create_static_alloc(def_id)) - } + ) -> InterpResult<'tcx, Pointer>; - /// Return the "base" tag for the given *global* allocation: the one that is used for direct - /// accesses to this static/const/fn allocation. If `id` is not a global allocation, - /// this will return an unusable tag (i.e., accesses will be UB)! + /// Return a "base" pointer for the given allocation: the one that is used for direct + /// accesses to this static/const/fn allocation, or the one returned from the heap allocator. /// - /// Called on the id returned by `thread_local_static_alloc_id` and `extern_static_alloc_id`, if needed. - /// - /// `offset` is relative inside the allocation. - fn tag_global_base_pointer( - memory_extra: &Self::MemoryExtra, + /// Not called on `extern` or thread-local statics (those use the methods above). + fn tag_alloc_base_pointer( + mem: &Memory<'mir, 'tcx, Self>, ptr: Pointer, ) -> Pointer; + /// "Int-to-pointer cast" + fn ptr_from_addr( + mem: &Memory<'mir, 'tcx, Self>, + addr: u64, + ) -> Pointer>; + + /// Convert a pointer with provenance into an allocation-offset pair. + fn ptr_get_alloc( + mem: &Memory<'mir, 'tcx, Self>, + ptr: Pointer, + ) -> (AllocId, Size); + /// Called to initialize the "extra" state of an allocation and make the pointers /// it contains (in relocations) tagged. The way we construct allocations is /// to always first construct it without extra and then add the extra. @@ -303,16 +309,13 @@ pub trait Machine<'mir, 'tcx>: Sized { /// allocation (because a copy had to be done to add tags or metadata), machine memory will /// cache the result. (This relies on `AllocMap::get_or` being able to add the /// owned allocation to the map even when the map is shared.) - /// - /// Also return the "base" tag to use for this allocation: the one that is used for direct - /// accesses to this allocation. If `kind == STATIC_KIND`, this tag must be consistent - /// with `tag_global_base_pointer`. fn init_allocation_extra<'b>( memory_extra: &Self::MemoryExtra, + tcx: TyCtxt<'tcx>, id: AllocId, alloc: Cow<'b, Allocation>, kind: Option>, - ) -> (Cow<'b, Allocation>, Self::PointerTag); + ) -> Cow<'b, Allocation>; /// Hook for performing extra checks on a memory read access. /// @@ -323,8 +326,8 @@ pub trait Machine<'mir, 'tcx>: Sized { fn memory_read( _memory_extra: &Self::MemoryExtra, _alloc_extra: &Self::AllocExtra, - _ptr: Pointer, - _size: Size, + _tag: Self::PointerTag, + _range: AllocRange, ) -> InterpResult<'tcx> { Ok(()) } @@ -334,8 +337,8 @@ pub trait Machine<'mir, 'tcx>: Sized { fn memory_written( _memory_extra: &mut Self::MemoryExtra, _alloc_extra: &mut Self::AllocExtra, - _ptr: Pointer, - _size: Size, + _tag: Self::PointerTag, + _range: AllocRange, ) -> InterpResult<'tcx> { Ok(()) } @@ -345,17 +348,8 @@ pub trait Machine<'mir, 'tcx>: Sized { fn memory_deallocated( _memory_extra: &mut Self::MemoryExtra, _alloc_extra: &mut Self::AllocExtra, - _ptr: Pointer, - _size: Size, - ) -> InterpResult<'tcx> { - Ok(()) - } - - /// Called after initializing static memory using the interpreter. - fn after_static_mem_initialized( - _ecx: &mut InterpCx<'mir, 'tcx, Self>, - _ptr: Pointer, - _size: Size, + _tag: Self::PointerTag, + _range: AllocRange, ) -> InterpResult<'tcx> { Ok(()) } @@ -400,19 +394,6 @@ pub trait Machine<'mir, 'tcx>: Sized { // By default, we do not support unwinding from panics Ok(StackPopJump::Normal) } - - /// "Int-to-pointer cast" - fn ptr_from_addr( - mem: &Memory<'mir, 'tcx, Self>, - addr: u64, - ) -> Pointer>; - - /// Convert a pointer with provenance into an allocation-offset pair, - /// or a `None` with an absolute address if that conversion is not possible. - fn ptr_get_alloc( - mem: &Memory<'mir, 'tcx, Self>, - ptr: Pointer, - ) -> (Option, Size); } // A lot of the flexibility above is just needed for `Miri`, but all "compile-time" machines @@ -461,17 +442,26 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) { #[inline(always)] fn init_allocation_extra<'b>( _memory_extra: &Self::MemoryExtra, - id: AllocId, + _tcx: TyCtxt<$tcx>, + _id: AllocId, alloc: Cow<'b, Allocation>, _kind: Option>, - ) -> (Cow<'b, Allocation>, Self::PointerTag) { + ) -> Cow<'b, Allocation> { // We do not use a tag so we can just cheaply forward the allocation - (alloc, id) + alloc + } + + fn extern_static_base_pointer( + mem: &Memory<$mir, $tcx, Self>, + def_id: DefId, + ) -> InterpResult<$tcx, Pointer> { + // Use the `AllocId` associated with the `DefId`. Any actual *access* will fail. + Ok(Pointer::new(mem.tcx.create_static_alloc(def_id), Size::ZERO)) } #[inline(always)] - fn tag_global_base_pointer( - _memory_extra: &Self::MemoryExtra, + fn tag_alloc_base_pointer( + _mem: &Memory<$mir, $tcx, Self>, ptr: Pointer, ) -> Pointer { ptr @@ -486,9 +476,9 @@ pub macro compile_time_machine(<$mir: lifetime, $tcx: lifetime>) { fn ptr_get_alloc( _mem: &Memory<$mir, $tcx, Self>, ptr: Pointer, - ) -> (Option, Size) { + ) -> (AllocId, Size) { // We know `offset` is relative to the allocation, so we can use `into_parts`. let (alloc_id, offset) = ptr.into_parts(); - (Some(alloc_id), offset) + (alloc_id, offset) } } diff --git a/compiler/rustc_mir/src/interpret/memory.rs b/compiler/rustc_mir/src/interpret/memory.rs index 5e5a969c26ab..aae3721e4e4f 100644 --- a/compiler/rustc_mir/src/interpret/memory.rs +++ b/compiler/rustc_mir/src/interpret/memory.rs @@ -168,20 +168,17 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { // We know `offset` is relative to the allocation, so we can use `into_parts`. let (alloc_id, offset) = ptr.into_parts(); // We need to handle `extern static`. - let alloc_id = match self.tcx.get_global_alloc(alloc_id) { + match self.tcx.get_global_alloc(alloc_id) { Some(GlobalAlloc::Static(def_id)) if self.tcx.is_thread_local_static(def_id) => { bug!("global memory cannot point to thread-local static") } Some(GlobalAlloc::Static(def_id)) if self.tcx.is_foreign_item(def_id) => { - M::extern_static_alloc_id(self, def_id)? + return M::extern_static_base_pointer(self, def_id); } - _ => { - // No need to change the `AllocId`. - alloc_id - } - }; + _ => {} + } // And we need to get the tag. - Ok(M::tag_global_base_pointer(&self.extra, Pointer::new(alloc_id, offset))) + Ok(M::tag_alloc_base_pointer(self, Pointer::new(alloc_id, offset))) } pub fn create_fn_alloc( @@ -236,9 +233,9 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { "dynamically allocating global memory" ); // This is a new allocation, not a new global one, so no `global_base_ptr`. - let (alloc, tag) = M::init_allocation_extra(&self.extra, id, Cow::Owned(alloc), Some(kind)); + let alloc = M::init_allocation_extra(&self.extra, self.tcx, id, Cow::Owned(alloc), Some(kind)); self.alloc_map.insert(id, (kind, alloc.into_owned())); - Pointer::new(tag, Size::ZERO) + M::tag_alloc_base_pointer(self, Pointer::new(id, Size::ZERO)) } pub fn reallocate( @@ -249,7 +246,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { new_align: Align, kind: MemoryKind, ) -> InterpResult<'tcx, Pointer> { - let (alloc_id, offset, ptr) = self.ptr_force_alloc(ptr)?; + let (alloc_id, offset, ptr) = self.ptr_get_alloc(ptr)?; if offset.bytes() != 0 { throw_ub_format!( "reallocating {:?} which does not point to the beginning of an object", @@ -284,7 +281,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { old_size_and_align: Option<(Size, Align)>, kind: MemoryKind, ) -> InterpResult<'tcx> { - let (alloc_id, offset, ptr) = self.ptr_force_alloc(ptr)?; + let (alloc_id, offset, ptr) = self.ptr_get_alloc(ptr)?; trace!("deallocating: {}", alloc_id); if offset.bytes() != 0 { @@ -337,7 +334,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { // Let the machine take some extra action let size = alloc.size(); - M::memory_deallocated(&mut self.extra, &mut alloc.extra, ptr, size)?; + M::memory_deallocated(&mut self.extra, &mut alloc.extra, ptr.provenance, alloc_range(Size::ZERO, size))?; // Don't forget to remember size and align of this now-dead allocation let old = self.dead_alloc_map.insert(alloc_id, (size, alloc.align)); @@ -424,7 +421,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { self.ptr_try_get_alloc(ptr) } else { // A "real" access, we insist on getting an `AllocId`. - Ok(self.ptr_force_alloc(ptr)?) + Ok(self.ptr_get_alloc(ptr)?) }; Ok(match ptr_or_addr { Err(addr) => { @@ -530,14 +527,13 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { M::before_access_global(memory_extra, id, alloc, def_id, is_write)?; let alloc = Cow::Borrowed(alloc); // We got tcx memory. Let the machine initialize its "extra" stuff. - let (alloc, tag) = M::init_allocation_extra( + let alloc = M::init_allocation_extra( memory_extra, + tcx, id, // always use the ID we got as input, not the "hidden" one. alloc, M::GLOBAL_KIND.map(MemoryKind::Machine), ); - // Sanity check that this is the same tag we would have gotten via `global_base_pointer`. - debug_assert!(tag == M::tag_global_base_pointer(memory_extra, id.into()).provenance); Ok(alloc) } @@ -596,8 +592,8 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { }, )?; if let Some((alloc_id, offset, ptr, alloc)) = ptr_and_alloc { - M::memory_read(&self.extra, &alloc.extra, ptr, size)?; let range = alloc_range(offset, size); + M::memory_read(&self.extra, &alloc.extra, ptr.provenance, range)?; Ok(Some(AllocRef { alloc, range, tcx: self.tcx, alloc_id })) } else { // Even in this branch we have to be sure that we actually access the allocation, in @@ -662,8 +658,8 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { // FIXME: can we somehow avoid looking up the allocation twice here? // We cannot call `get_raw_mut` inside `check_and_deref_ptr` as that would duplicate `&mut self`. let (alloc, extra) = self.get_raw_mut(alloc_id)?; - M::memory_written(extra, &mut alloc.extra, ptr, size)?; let range = alloc_range(offset, size); + M::memory_written(extra, &mut alloc.extra, ptr.provenance, range)?; Ok(Some(AllocRefMut { alloc, range, tcx, alloc_id })) } else { Ok(None) @@ -756,7 +752,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { &self, ptr: Pointer>, ) -> InterpResult<'tcx, FnVal<'tcx, M::ExtraFnVal>> { - let (alloc_id, offset, ptr) = self.ptr_force_alloc(ptr)?; + let (alloc_id, offset, ptr) = self.ptr_get_alloc(ptr)?; if offset.bytes() != 0 { throw_ub!(InvalidFunctionPointer(ptr.erase_for_fmt())) } @@ -1036,7 +1032,8 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { Some(src_ptr) => src_ptr, }; let src_alloc = self.get_raw(src_alloc_id)?; - M::memory_read(&self.extra, &src_alloc.extra, src, size)?; + let src_range = alloc_range(src_offset, size); + M::memory_read(&self.extra, &src_alloc.extra, src.provenance, src_range)?; // We need the `dest` ptr for the next operation, so we get it now. // We already did the source checks and called the hooks so we are good to return early. let (dest_alloc_id, dest_offset, dest) = match dest_parts { @@ -1051,23 +1048,24 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { // relocations overlapping the edges; those would not be handled correctly). let relocations = src_alloc.prepare_relocation_copy( self, - alloc_range(src_offset, size), + src_range, dest_offset, num_copies, ); // Prepare a copy of the initialization mask. - let compressed = src_alloc.compress_uninit_range(alloc_range(src_offset, size)); + let compressed = src_alloc.compress_uninit_range(src_range); // This checks relocation edges on the src. let src_bytes = src_alloc - .get_bytes_with_uninit_and_ptr(&tcx, alloc_range(src_offset, size)) + .get_bytes_with_uninit_and_ptr(&tcx, src_range) .map_err(|e| e.to_interp_error(src_alloc_id))? .as_ptr(); // raw ptr, so we can also get a ptr to the destination allocation // Destination alloc preparations and access hooks. let (dest_alloc, extra) = self.get_raw_mut(dest_alloc_id)?; - M::memory_written(extra, &mut dest_alloc.extra, dest, size * num_copies)?; + let dest_range = alloc_range(dest_offset, size * num_copies); + M::memory_written(extra, &mut dest_alloc.extra, dest.provenance, dest_range)?; let dest_bytes = dest_alloc - .get_bytes_mut_ptr(&tcx, alloc_range(dest_offset, size * num_copies)) + .get_bytes_mut_ptr(&tcx, dest_range) .as_mut_ptr(); if compressed.no_bytes_init() { @@ -1077,7 +1075,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { // This also avoids writing to the target bytes so that the backing allocation is never // touched if the bytes stay uninitialized for the whole interpreter execution. On contemporary // operating system this can avoid physically allocating the page. - dest_alloc.mark_init(alloc_range(dest_offset, size * num_copies), false); // `Size` multiplication + dest_alloc.mark_init(dest_range, false); // `Size` multiplication dest_alloc.mark_relocation_range(relocations); return Ok(()); } @@ -1119,7 +1117,7 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { // now fill in all the "init" data dest_alloc.mark_compressed_init_range( &compressed, - alloc_range(dest_offset, size), + alloc_range(dest_offset, size), // just a single copy (i.e., not full `dest_range`) num_copies, ); // copy the relocations to the destination @@ -1141,29 +1139,24 @@ impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { } } - /// Internal helper for turning a "maybe pointer" into a proper pointer (and some information + /// Turning a "maybe pointer" into a proper pointer (and some information /// about where it points), or an absolute address. - pub(super) fn ptr_try_get_alloc( + pub fn ptr_try_get_alloc( &self, ptr: Pointer>, ) -> Result<(AllocId, Size, Pointer), u64> { - match ptr.into_pointer_or_offset() { + match ptr.into_pointer_or_addr() { Ok(ptr) => { let (alloc_id, offset) = M::ptr_get_alloc(self, ptr); - if let Some(alloc_id) = alloc_id { - Ok((alloc_id, offset, ptr)) - } else { - Err(offset.bytes()) - } + Ok((alloc_id, offset, ptr)) } - Err(offset) => Err(offset.bytes()), + Err(addr) => Err(addr.bytes()), } } - /// Internal helper for turning a "maybe pointer" into a proper pointer (and some information - /// about where it points). + /// Turning a "maybe pointer" into a proper pointer (and some information about where it points). #[inline(always)] - pub(super) fn ptr_force_alloc( + pub fn ptr_get_alloc( &self, ptr: Pointer>, ) -> InterpResult<'tcx, (AllocId, Size, Pointer)> { diff --git a/compiler/rustc_mir/src/interpret/place.rs b/compiler/rustc_mir/src/interpret/place.rs index 1c9905b775f0..5b0a940637d1 100644 --- a/compiler/rustc_mir/src/interpret/place.rs +++ b/compiler/rustc_mir/src/interpret/place.rs @@ -199,6 +199,11 @@ impl MemPlace { MemPlace { ptr, align, meta: MemPlaceMeta::None } } + /// Adjust the provenance of the main pointer (metadata is unaffected). + pub fn map_provenance(self, f: impl FnOnce(Option) -> Option) -> Self { + MemPlace { ptr: self.ptr.map_provenance(f), ..self } + } + /// Turn a mplace into a (thin or wide) pointer, as a reference, pointing to the same space. /// This is the inverse of `ref_to_mplace`. #[inline(always)] @@ -252,7 +257,7 @@ impl<'tcx, Tag: Copy> MPlaceTy<'tcx, Tag> { } #[inline] - fn from_aligned_ptr(ptr: Pointer>, layout: TyAndLayout<'tcx>) -> Self { + pub fn from_aligned_ptr(ptr: Pointer>, layout: TyAndLayout<'tcx>) -> Self { MPlaceTy { mplace: MemPlace::from_ptr(ptr, layout.align.abi), layout } } @@ -695,16 +700,6 @@ where Ok(place_ty) } - /// Write a scalar to a place - #[inline(always)] - pub fn write_scalar( - &mut self, - val: impl Into>, - dest: &PlaceTy<'tcx, M::PointerTag>, - ) -> InterpResult<'tcx> { - self.write_immediate(Immediate::Scalar(val.into()), dest) - } - /// Write an immediate to a place #[inline(always)] pub fn write_immediate( @@ -722,21 +717,24 @@ where Ok(()) } - /// Write an `Immediate` to memory. + /// Write a scalar to a place #[inline(always)] - pub fn write_immediate_to_mplace( + pub fn write_scalar( &mut self, - src: Immediate, - dest: &MPlaceTy<'tcx, M::PointerTag>, + val: impl Into>, + dest: &PlaceTy<'tcx, M::PointerTag>, ) -> InterpResult<'tcx> { - self.write_immediate_to_mplace_no_validate(src, dest)?; + self.write_immediate(Immediate::Scalar(val.into()), dest) + } - if M::enforce_validity(self) { - // Data got changed, better make sure it matches the type! - self.validate_operand(&dest.into())?; - } - - Ok(()) + /// Write a pointer to a place + #[inline(always)] + pub fn write_pointer( + &mut self, + ptr: impl Into>>, + dest: &PlaceTy<'tcx, M::PointerTag>, + ) -> InterpResult<'tcx> { + self.write_scalar(Scalar::from_maybe_pointer(ptr.into(), self), dest) } /// Write an immediate to a place. diff --git a/compiler/rustc_mir/src/interpret/step.rs b/compiler/rustc_mir/src/interpret/step.rs index f0787ad0c660..f2a8a067dfac 100644 --- a/compiler/rustc_mir/src/interpret/step.rs +++ b/compiler/rustc_mir/src/interpret/step.rs @@ -162,9 +162,8 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { use rustc_middle::mir::Rvalue::*; match *rvalue { ThreadLocalRef(did) => { - let id = M::thread_local_static_alloc_id(self, did)?; - let val = self.global_base_pointer(id.into())?; - self.write_scalar(Scalar::from_pointer(val, &*self.tcx), &dest)?; + let ptr = M::thread_local_static_base_pointer(self, did)?; + self.write_pointer(ptr, &dest)?; } Use(ref operand) => { diff --git a/compiler/rustc_mir/src/transform/const_prop.rs b/compiler/rustc_mir/src/transform/const_prop.rs index 73bbb38a93a9..5e2c47be3a2e 100644 --- a/compiler/rustc_mir/src/transform/const_prop.rs +++ b/compiler/rustc_mir/src/transform/const_prop.rs @@ -869,7 +869,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { let alloc = this .ecx .intern_with_temp_alloc(value.layout, |ecx, dest| { - ecx.write_immediate_to_mplace(*imm, dest) + ecx.write_immediate(*imm, dest) }) .unwrap(); Ok(Some(alloc))