diff --git a/clippy_utils/src/qualify_min_const_fn.rs b/clippy_utils/src/qualify_min_const_fn.rs
index 78d8f1e213af..283b20fc24d8 100644
--- a/clippy_utils/src/qualify_min_const_fn.rs
+++ b/clippy_utils/src/qualify_min_const_fn.rs
@@ -125,16 +125,11 @@ fn check_rvalue<'tcx>(
         Rvalue::Len(place) | Rvalue::Discriminant(place) | Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => {
             check_place(tcx, *place, span, body)
         },
-        Rvalue::Cast(CastKind::Misc, operand, cast_ty) => {
-            use rustc_middle::ty::cast::CastTy;
-            let cast_in = CastTy::from_ty(operand.ty(body, tcx)).expect("bad input type for cast");
-            let cast_out = CastTy::from_ty(*cast_ty).expect("bad output type for cast");
-            match (cast_in, cast_out) {
-                (CastTy::Ptr(_) | CastTy::FnPtr, CastTy::Int(_)) => {
-                    Err((span, "casting pointers to ints is unstable in const fn".into()))
-                },
-                _ => check_operand(tcx, operand, span, body),
-            }
+        Rvalue::Cast(CastKind::PointerAddress, _, _) => {
+            Err((span, "casting pointers to ints is unstable in const fn".into()))
+        },
+        Rvalue::Cast(CastKind::Misc, operand, _) => {
+            check_operand(tcx, operand, span, body)
         },
         Rvalue::Cast(CastKind::Pointer(PointerCast::MutToConstPointer | PointerCast::ArrayToPointer), operand, _) => {
             check_operand(tcx, operand, span, body)