add graceful shim for the custom O_TMPFILE linux file opening flag plus test case

src/tools/miri/src/shims/unix/fs.rs

@@ -605,6 +605,15 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
             // (Technically we do not support *not* setting this flag, but we ignore that.)
             mirror |= o_cloexec;
         }
+        if this.tcx.sess.target.os == "linux" {
+            let o_tmpfile = this.eval_libc_i32("O_TMPFILE")?;
+            if flag & o_tmpfile != 0 {
+                // if the flag contains `O_TMPFILE` then we return a graceful error
+                let eopnotsupp = this.eval_libc("EOPNOTSUPP")?;
+                this.set_last_error(eopnotsupp)?;
+                return Ok(-1);
+            }
+        }
         // If `flag` is not equal to `mirror`, there is an unsupported option enabled in `flag`,
         // then we throw an error.
         if flag != mirror {

src/tools/miri/tests/pass-dep/shims/libc-fs.rs

@@ -6,7 +6,7 @@
 
 use std::convert::TryInto;
 use std::ffi::CString;
-use std::fs::{canonicalize, remove_file, File};
+use std::fs::{canonicalize, remove_dir_all, remove_file, File};
 use std::io::{Error, ErrorKind, Write};
 use std::os::unix::ffi::OsStrExt;
 use std::path::PathBuf;
@@ -18,6 +18,8 @@ fn main() {
     test_file_open_unix_allow_two_args();
     test_file_open_unix_needs_three_args();
     test_file_open_unix_extra_third_arg();
+    #[cfg(target_os = "linux")]
+    test_o_tmpfile_flag();
 }
 
 fn tmp() -> PathBuf {
@@ -45,6 +47,15 @@ fn prepare(filename: &str) -> PathBuf {
     path
 }
 
+/// Prepare directory: compute directory name and make sure it does not exist.
+#[allow(unused)]
+fn prepare_dir(dirname: &str) -> PathBuf {
+    let path = tmp().join(&dirname);
+    // Clean the directory for robustness.
+    remove_dir_all(&path).ok();
+    path
+}
+
 /// Prepare like above, and also write some initial content to the file.
 fn prepare_with_content(filename: &str, content: &[u8]) -> PathBuf {
     let path = prepare(filename);
@@ -135,3 +146,22 @@ fn test_readlink() {
     assert_eq!(res, -1);
     assert_eq!(Error::last_os_error().kind(), ErrorKind::NotFound);
 }
+
+#[cfg(target_os = "linux")]
+fn test_o_tmpfile_flag() {
+    use std::fs::{create_dir, OpenOptions};
+    use std::os::unix::fs::OpenOptionsExt;
+    let dir_path = prepare_dir("miri_test_fs_dir");
+    create_dir(&dir_path).unwrap();
+    // test that the `O_TMPFILE` custom flag gracefully errors instead of stopping execution
+    assert_eq!(
+        Some(libc::EOPNOTSUPP),
+        OpenOptions::new()
+            .read(true)
+            .write(true)
+            .custom_flags(libc::O_TMPFILE)
+            .open(dir_path)
+            .unwrap_err()
+            .raw_os_error(),
+    );
+}