diff --git a/src/tools/miri/src/borrow_tracker/tree_borrows/tree.rs b/src/tools/miri/src/borrow_tracker/tree_borrows/tree.rs
index 1f29bcfc2b03..22bd63bd6b6f 100644
--- a/src/tools/miri/src/borrow_tracker/tree_borrows/tree.rs
+++ b/src/tools/miri/src/borrow_tracker/tree_borrows/tree.rs
@@ -756,6 +756,8 @@ impl<'tcx> Tree {
                             // Don't check for protector if it is a Cell (see `unsafe_cell_deallocate` in `interior_mutability.rs`).
                             // Related to https://github.com/rust-lang/rust/issues/55005.
                             && !perm.permission().is_cell()
+                            // Only trigger UB if the accessed bit is set, i.e. if the protector is actually protecting this offset. See #4579.
+                            && perm.is_accessed()
                         {
                             Err(TransitionError::ProtectedDealloc)
                         } else {
diff --git a/src/tools/miri/tests/fail/both_borrows/zero-sized-protected.rs b/src/tools/miri/tests/fail/both_borrows/zero-sized-protected.rs
deleted file mode 100644
index df9a73a444ed..000000000000
--- a/src/tools/miri/tests/fail/both_borrows/zero-sized-protected.rs
+++ /dev/null
@@ -1,18 +0,0 @@
-//@revisions: stack tree
-//@[tree]compile-flags: -Zmiri-tree-borrows
-use std::alloc::{Layout, alloc, dealloc};
-
-// `x` is strongly protected but covers zero bytes.
-// Let's see if deallocating the allocation x points to is UB:
-// in TB, it is UB, but in SB it is not.
-fn test(_x: &mut (), ptr: *mut u8, l: Layout) {
-    unsafe { dealloc(ptr, l) }; //~[tree] ERROR: /deallocation .* is forbidden/
-}
-
-fn main() {
-    let l = Layout::from_size_align(1, 1).unwrap();
-    let ptr = unsafe { alloc(l) };
-    unsafe { test(&mut *ptr.cast::<()>(), ptr, l) };
-    // In SB the test would pass if it weren't for this line.
-    unsafe { std::hint::unreachable_unchecked() }; //~[stack] ERROR: unreachable
-}
diff --git a/src/tools/miri/tests/fail/both_borrows/zero-sized-protected.stack.stderr b/src/tools/miri/tests/fail/both_borrows/zero-sized-protected.stack.stderr
deleted file mode 100644
index 3e4a7ccac363..000000000000
--- a/src/tools/miri/tests/fail/both_borrows/zero-sized-protected.stack.stderr
+++ /dev/null
@@ -1,15 +0,0 @@
-error: Undefined Behavior: entering unreachable code
-  --> tests/fail/both_borrows/zero-sized-protected.rs:LL:CC
-   |
-LL |     unsafe { std::hint::unreachable_unchecked() };
-   |              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Undefined Behavior occurred here
-   |
-   = help: this indicates a bug in the program: it performed an invalid operation, and caused Undefined Behavior
-   = help: see https://doc.rust-lang.org/nightly/reference/behavior-considered-undefined.html for further information
-   = note: BACKTRACE:
-   = note: inside `main` at tests/fail/both_borrows/zero-sized-protected.rs:LL:CC
-
-note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
-
-error: aborting due to 1 previous error
-
diff --git a/src/tools/miri/tests/fail/both_borrows/zero-sized-protected.tree.stderr b/src/tools/miri/tests/fail/both_borrows/zero-sized-protected.tree.stderr
deleted file mode 100644
index 66a7d7794e96..000000000000
--- a/src/tools/miri/tests/fail/both_borrows/zero-sized-protected.tree.stderr
+++ /dev/null
@@ -1,32 +0,0 @@
-error: Undefined Behavior: deallocation through <TAG> (root of the allocation) at ALLOC[0x0] is forbidden
-  --> tests/fail/both_borrows/zero-sized-protected.rs:LL:CC
-   |
-LL |     unsafe { dealloc(ptr, l) };
-   |              ^^^^^^^^^^^^^^^ Undefined Behavior occurred here
-   |
-   = help: this indicates a potential bug in the program: it performed an invalid operation, but the Tree Borrows rules it violated are still experimental
-   = help: see https://github.com/rust-lang/unsafe-code-guidelines/blob/master/wip/tree-borrows.md for further information
-   = help: the allocation of the accessed tag <TAG> (root of the allocation) also contains the strongly protected tag <TAG>
-   = help: the strongly protected tag <TAG> disallows deallocations
-help: the accessed tag <TAG> was created here
-  --> tests/fail/both_borrows/zero-sized-protected.rs:LL:CC
-   |
-LL |     let ptr = unsafe { alloc(l) };
-   |                        ^^^^^^^^
-help: the strongly protected tag <TAG> was created here, in the initial state Reserved
-  --> tests/fail/both_borrows/zero-sized-protected.rs:LL:CC
-   |
-LL | fn test(_x: &mut (), ptr: *mut u8, l: Layout) {
-   |         ^^
-   = note: BACKTRACE (of the first span):
-   = note: inside `test` at tests/fail/both_borrows/zero-sized-protected.rs:LL:CC
-note: inside `main`
-  --> tests/fail/both_borrows/zero-sized-protected.rs:LL:CC
-   |
-LL |     unsafe { test(&mut *ptr.cast::<()>(), ptr, l) };
-   |              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-note: some details are omitted, run with `MIRIFLAGS=-Zmiri-backtrace=full` for a verbose backtrace
-
-error: aborting due to 1 previous error
-
diff --git a/src/tools/miri/tests/pass/both_borrows/basic_aliasing_model.rs b/src/tools/miri/tests/pass/both_borrows/basic_aliasing_model.rs
index 82976326a8df..115e232dde4c 100644
--- a/src/tools/miri/tests/pass/both_borrows/basic_aliasing_model.rs
+++ b/src/tools/miri/tests/pass/both_borrows/basic_aliasing_model.rs
@@ -1,6 +1,7 @@
 //@revisions: stack tree
 //@[tree]compile-flags: -Zmiri-tree-borrows
 #![feature(allocator_api)]
+use std::alloc::{Layout, alloc, dealloc};
 use std::cell::Cell;
 use std::ptr;
 
@@ -305,5 +306,14 @@ fn zst() {
         let ptr = &raw mut *b as *mut ();
         drop(b);
         let _ref = &mut *ptr;
+
+        // zero-sized protectors do not affect deallocation
+        fn with_protector(_x: &mut (), ptr: *mut u8, l: Layout) {
+            // `_x` here is strongly protected but covers zero bytes.
+            unsafe { dealloc(ptr, l) };
+        }
+        let l = Layout::from_size_align(1, 1).unwrap();
+        let ptr = alloc(l);
+        with_protector(&mut *ptr.cast::<()>(), ptr, l);
     }
 }